🧭 Classify processing faults in production dashboard

This commit is contained in:
d-robotics 2026-10-01 00:37:11 -07:00
parent 7ee9efec6c
commit 9c3dc37807
10 changed files with 177 additions and 35 deletions

View File

@ -155,18 +155,30 @@ def _meta_frame_count(metadata: dict[str, Any], video_path: str) -> int:
if identity is None:
raise RuntimeError(f"采集 meta 无法识别视频身份: {video_path}")
candidates = metadata.get("streams", {}).get("color_" + identity[1], {}).get("segments", [])
records = [item for item in candidates if isinstance(item, dict)
and str(item.get("index", "")).isdigit() and int(item["index"]) == identity[2]]
indexed_records = [item for item in candidates if isinstance(item, dict)
and str(item.get("index", "")).isdigit() and int(item["index"]) == identity[2]]
if not indexed_records:
raise RuntimeError(f"采集 meta 缺少对应相机、part 的分段记录: {video_path}")
records = indexed_records
# Old meta may contain index/count only. Keep that path only when none of
# this part's records provides a filename; never use anonymous records to
# override an explicit but different Capture/camera/file identity.
if any(item.get("file") or item.get("pts_file") for item in records):
records = [item for item in records
if (item.get("file") or item.get("pts_file"))
and all(isinstance(value, str) and _segment_identity(value) == identity
for value in (item.get("file"), item.get("pts_file")) if value)]
matched_records = [
item
for item in records
if (item.get("file") or item.get("pts_file"))
and all(
isinstance(value, str) and _segment_identity(value) == identity
for value in (item.get("file"), item.get("pts_file"))
if value
)
]
if not matched_records:
raise RuntimeError(f"采集 meta 文件身份冲突(匹配失败): {video_path},存在同 part 但不属于当前采集或相机的记录")
records = matched_records
if not records:
raise RuntimeError(f"采集 meta 帧数记录匹配失败: {video_path},未找到对应采集、相机和 part")
raise RuntimeError(f"采集 meta 缺少对应相机、part 的分段记录: {video_path}")
if len(records) != 1:
raise RuntimeError(f"采集 meta 帧数记录不唯一: {video_path},匹配 {len(records)} 条")
count = records[0].get("frame_count")

View File

@ -99,6 +99,7 @@ func (s *server) handleProductionDashboard(w http.ResponseWriter, r *http.Reques
q.ProjectID = params.Get("projectId")
q.SupplierID = params.Get("supplierId")
q.Group = params.Get("group")
q.ExceptionCategory = params.Get("exceptionCategory")
if q.Group == "" {
q.Group = "project"
}

View File

@ -10,6 +10,34 @@ import (
"time"
)
// exceptionCategory keeps operational failures actionable without changing the
// source-level attention total. It is derived from persisted failure text for
// compatibility with older runs that do not have a structured failure code.
func exceptionCategory(r domain.RunRecord) string {
message := r.Error
if r.Failure != nil {
message = r.Failure.Message + "\n" + message
}
switch {
case strings.Contains(message, "采集 meta 文件身份冲突"):
return "metadata_identity_conflict"
case strings.Contains(message, "采集 meta 缺少对应相机、part 的分段记录"):
return "metadata_pending"
case strings.Contains(message, "PTS") && (strings.Contains(message, "无效") || strings.Contains(message, "未严格递增")):
return "source_pts"
case strings.Contains(message, "meta 与 PTS"):
return "metadata_pts_mismatch"
case strings.Contains(message, "PreconditionFailed"):
return "source_changed"
case strings.Contains(message, "采集 meta 帧数记录匹配失败") || strings.Contains(message, "采集 meta 帧数记录不唯一") || strings.Contains(message, "采集 meta 帧数记录无效"):
return "metadata_identity"
case strings.Contains(message, "采集 meta 缺少对应相机、part"):
return "legacy_metadata_identity"
default:
return "other"
}
}
func sourceKey(c domain.CaptureRecord) string {
prefix := c.SourcePrefix
if prefix == "" {
@ -330,6 +358,9 @@ func Build(d Snapshot) Report {
if r.Status == domain.RunStatusFailed || (root(c.RunID) == r.ID && c.Status == domain.CaptureRecordFailed) {
s.Exception = true
}
if s.Exception {
s.ExceptionCategory = exceptionCategory(r)
}
if s.SupplierID == Unknown || s.Stage == "unknown" {
result.Incomplete = true
}

View File

@ -48,6 +48,52 @@ func TestOnlyOperationalFailuresCountAsExceptions(t *testing.T) {
})
}
}
func TestOperationalFailureCategories(t *testing.T) {
for _, tc := range []struct {
name, message, want string
}{
{"legacy metadata", "采集 meta 缺少对应相机、part 的帧数记录", "legacy_metadata_identity"},
{"metadata identity", "采集 meta 帧数记录匹配失败", "metadata_identity"},
{"pts", "PTS 第 10923 行无效或未严格递增", "source_pts"},
{"meta pts", "采集 meta 与 PTS 数量不一致", "metadata_pts_mismatch"},
{"source changed", "PreconditionFailed", "source_changed"},
{"metadata pending", "采集 meta 缺少对应相机、part 的分段记录", "metadata_pending"},
{"metadata identity conflict", "采集 meta 文件身份冲突", "metadata_identity_conflict"},
} {
t.Run(tc.name, func(t *testing.T) {
d := Snapshot{
Captures: []domain.CaptureRecord{{ID: "c", ProjectID: "p", Prefix: "c", RunID: "r"}},
Runs: []domain.RunRecord{{ID: "r", TaskID: "c", Status: domain.RunStatusFailed, Error: tc.message}},
}
r := Build(d)
if len(r.Sources) != 1 || r.Sources[0].ExceptionCategory != tc.want {
t.Fatalf("category=%q want %q source=%+v", r.Sources[0].ExceptionCategory, tc.want, r.Sources[0])
}
if got := Summarize(r, Query{View: "current"}).ExceptionCategories[tc.want]; got != 1 {
t.Fatalf("category summary=%v", Summarize(r, Query{View: "current"}))
}
})
}
}
func TestExceptionCategoryFilterKeepsTotalsAndDetailsAligned(t *testing.T) {
r := Report{Sources: []Source{
{ID: "pending", ProjectID: "p", Stage: "intake", Exception: true, ExceptionCategory: "metadata_pending"},
{ID: "pts", ProjectID: "p", Stage: "intake", Exception: true, ExceptionCategory: "source_pts"},
{ID: "ok", ProjectID: "p", Stage: "annotation"},
}}
q := Query{View: "current", ExceptionCategory: "metadata_pending"}
summary := Summarize(r, q)
if summary.Totals["total"] != 1 || summary.Totals["attention"] != 1 || summary.ExceptionCategories["metadata_pending"] != 1 {
t.Fatalf("summary=%+v", summary)
}
items := Details(r, q, "attention", "")
if len(items) != 1 || items[0].ID != "pending" {
t.Fatalf("details=%+v", items)
}
}
func TestMixedPartOwnershipDoesNotCreateFault(t *testing.T) {
d := Snapshot{}
for n, owner := range []string{"vendor", ""} {

View File

@ -26,19 +26,20 @@ type Task struct {
Status string `json:"status"`
}
type Source struct {
ID string `json:"id"`
ProjectID string `json:"projectId"`
Name string `json:"name"`
WorkflowID string `json:"workflowId,omitempty"`
RunID string `json:"runId,omitempty"`
SupplierID string `json:"supplierId"`
Stage string `json:"stage"`
Detail string `json:"detail"`
Exception bool `json:"exception"`
DiscoveredAt string `json:"discoveredAt"`
Tasks []Task `json:"tasks"`
Runs []RunReference `json:"runs"`
Evidence []Evidence `json:"evidence,omitempty"`
ID string `json:"id"`
ProjectID string `json:"projectId"`
Name string `json:"name"`
WorkflowID string `json:"workflowId,omitempty"`
RunID string `json:"runId,omitempty"`
SupplierID string `json:"supplierId"`
Stage string `json:"stage"`
Detail string `json:"detail"`
Exception bool `json:"exception"`
ExceptionCategory string `json:"exceptionCategory,omitempty"`
DiscoveredAt string `json:"discoveredAt"`
Tasks []Task `json:"tasks"`
Runs []RunReference `json:"runs"`
Evidence []Evidence `json:"evidence,omitempty"`
}
type Event struct {
ID, SourceID, SupplierID, Metric, At string

View File

@ -9,6 +9,7 @@ import (
type Query struct {
View, ProjectID, SupplierID, Group string
ExceptionCategory string
From, To time.Time
}
type Row struct {
@ -16,11 +17,12 @@ type Row struct {
Counts map[string]int `json:"counts"`
}
type Summary struct {
Totals map[string]int `json:"totals"`
StageErrors map[string]int `json:"stageErrors"`
Rows []Row `json:"rows"`
SupplierOverlap bool `json:"supplierOverlap"`
Incomplete bool `json:"incomplete"`
Totals map[string]int `json:"totals"`
StageErrors map[string]int `json:"stageErrors"`
ExceptionCategories map[string]int `json:"exceptionCategories"`
Rows []Row `json:"rows"`
SupplierOverlap bool `json:"supplierOverlap"`
Incomplete bool `json:"incomplete"`
}
func ParseQuery(view, from, to string, now time.Time) (Query, error) {
@ -59,7 +61,7 @@ func (q Query) eventInRange(e Event) bool {
return err == nil && !at.Before(q.From) && at.Before(q.To)
}
func Summarize(r Report, q Query) Summary {
out := Summary{Totals: map[string]int{}, StageErrors: map[string]int{}, Rows: []Row{}, Incomplete: r.Incomplete}
out := Summary{Totals: map[string]int{}, StageErrors: map[string]int{}, ExceptionCategories: map[string]int{}, Rows: []Row{}, Incomplete: r.Incomplete}
rows := map[string]map[string]int{}
sources := map[string]Source{}
for _, s := range r.Sources {
@ -79,6 +81,9 @@ func Summarize(r Report, q Query) Summary {
if q.SupplierID != "" && s.SupplierID != q.SupplierID {
continue
}
if q.ExceptionCategory != "" && (!s.Exception || s.ExceptionCategory != q.ExceptionCategory) {
continue
}
row := s.ProjectID
if q.Group == "supplier" {
row = s.SupplierID
@ -90,6 +95,11 @@ func Summarize(r Report, q Query) Summary {
if s.Exception {
out.Totals["attention"]++
out.StageErrors[s.Stage]++
category := s.ExceptionCategory
if category == "" {
category = "other"
}
out.ExceptionCategories[category]++
add(row, "attention")
}
}
@ -167,6 +177,9 @@ func Details(r Report, q Query, metric, search string) []Source {
if q.SupplierID != "" && q.SupplierID != s.SupplierID {
continue
}
if q.ExceptionCategory != "" && (!s.Exception || s.ExceptionCategory != q.ExceptionCategory) {
continue
}
if metric == "attention" {
if !s.Exception {
continue

View File

@ -141,7 +141,7 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
}
}
runFilter := organizationScopedFilter(org, bson.M{"$or": bson.A{bson.M{"projectId": bson.M{"$in": projects}}, bson.M{"taskId": bson.M{"$in": captureIDs}}, bson.M{"id": bson.M{"$in": runIDs}}}})
if err := read(s.runs, runFilter, fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome"), &d.Runs); err != nil {
if err := read(s.runs, runFilter, fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error"), &d.Runs); err != nil {
return d, err
}
if err := read(s.humanItems, filter, fields("id", "projectId", "supplierId", "runId", "batchId", "legacyTaskId", "currentStageId", "status", "pendingCombinedSubmissionId", "releasePartitionId", "processingJob.status"), &d.Items); err != nil {

View File

@ -155,18 +155,30 @@ def _meta_frame_count(metadata: dict[str, Any], video_path: str) -> int:
if identity is None:
raise RuntimeError(f"采集 meta 无法识别视频身份: {video_path}")
candidates = metadata.get("streams", {}).get("color_" + identity[1], {}).get("segments", [])
records = [item for item in candidates if isinstance(item, dict)
and str(item.get("index", "")).isdigit() and int(item["index"]) == identity[2]]
indexed_records = [item for item in candidates if isinstance(item, dict)
and str(item.get("index", "")).isdigit() and int(item["index"]) == identity[2]]
if not indexed_records:
raise RuntimeError(f"采集 meta 缺少对应相机、part 的分段记录: {video_path}")
records = indexed_records
# Old meta may contain index/count only. Keep that path only when none of
# this part's records provides a filename; never use anonymous records to
# override an explicit but different Capture/camera/file identity.
if any(item.get("file") or item.get("pts_file") for item in records):
records = [item for item in records
if (item.get("file") or item.get("pts_file"))
and all(isinstance(value, str) and _segment_identity(value) == identity
for value in (item.get("file"), item.get("pts_file")) if value)]
matched_records = [
item
for item in records
if (item.get("file") or item.get("pts_file"))
and all(
isinstance(value, str) and _segment_identity(value) == identity
for value in (item.get("file"), item.get("pts_file"))
if value
)
]
if not matched_records:
raise RuntimeError(f"采集 meta 文件身份冲突(匹配失败): {video_path},存在同 part 但不属于当前采集或相机的记录")
records = matched_records
if not records:
raise RuntimeError(f"采集 meta 帧数记录匹配失败: {video_path},未找到对应采集、相机和 part")
raise RuntimeError(f"采集 meta 缺少对应相机、part 的分段记录: {video_path}")
if len(records) != 1:
raise RuntimeError(f"采集 meta 帧数记录不唯一: {video_path},匹配 {len(records)} 条")
count = records[0].get("frame_count")

View File

@ -21,6 +21,16 @@ const metrics: Record<string, string> = {
delivered: "成功交付",
rejected: "退回次数",
};
const exceptionCategories: Record<string, string> = {
legacy_metadata_identity: "历史元数据匹配",
metadata_pending: "元数据待补齐",
metadata_identity_conflict: "元数据身份冲突",
metadata_identity: "元数据待核对",
metadata_pts_mismatch: "元数据与 PTS 不一致",
source_pts: "源时间戳故障",
source_changed: "源文件已变化",
other: "其他处理故障",
};
const control = "rounded-lg border border-[color:var(--app-border)] bg-[color:var(--app-panel)] px-3 py-2 text-sm";
const beijingDate = (offset = 0) => new Date(Date.now() + 8 * 3600000 + offset * 86400000).toISOString().slice(0, 10);
export function ProductionStatistics() {
@ -309,6 +319,20 @@ export function ProductionStatistics() {
</button>
))}
</div>
{view === "current" && Object.entries(result.summary.exceptionCategories || {}).length > 0 && (
<div className="flex flex-wrap items-center gap-2 text-sm">
<span className="text-gray-500">故障分类:</span>
{Object.entries(result.summary.exceptionCategories).map(([id, count]) => (
<button
key={id}
className={`${control} text-red-700`}
onClick={() => open("attention", { exceptionCategory: id })}
>
{exceptionCategories[id] || id} {count}
</button>
))}
</div>
)}
{view === "period" && result.summary.incomplete && (
<p className="text-sm text-amber-700">
历史记录不完整,展示可核实的完成记录;缺失记录不代表实际产出为零。
@ -440,7 +464,7 @@ export function ProductionStatistics() {
<p className="my-2 text-sm text-gray-500">
当前状态:{names.get(s.projectId) || s.projectId} · {names.get(s.supplierId) || s.supplierId} ·{" "}
{stages[s.stage]}
{s.exception ? " · 处理故障" : ""}
{s.exception ? ` · ${exceptionCategories[s.exceptionCategory || "other"] || "处理故障"}` : ""}
</p>
{(s.runs || [])
.filter((run) => run.workflowId)

View File

@ -11,6 +11,7 @@ export type ProductionSource = {
stage: string;
detail: string;
exception: boolean;
exceptionCategory?: string;
discoveredAt: string;
tasks: { id: string; stage: string; status: string }[];
runs: { id: string; workflowId: string; part: string }[];
@ -33,6 +34,7 @@ export type ProductionResult = {
summary: {
totals: Record<string, number>;
stageErrors: Record<string, number>;
exceptionCategories: Record<string, number>;
rows: { id: string; counts: Record<string, number> }[];
supplierOverlap: boolean;
incomplete: boolean;