forked from longtao.wu/droboticflow
482 lines
15 KiB
Go
482 lines
15 KiB
Go
package productionstats
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"droboticflow/internal/domain"
|
|
"fmt"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// exceptionCategory keeps operational failures actionable without changing the
|
|
// source-level attention total. It is derived from persisted failure text for
|
|
// compatibility with older runs that do not have a structured failure code.
|
|
func ExceptionCategory(r domain.RunRecord) string { return exceptionCategory(r) }
|
|
func exceptionCategory(r domain.RunRecord) string {
|
|
switch r.QueryExceptionCategory {
|
|
case "metadata_identity_conflict", "metadata_pending", "source_pts", "metadata_pts_mismatch", "source_changed", "metadata_identity", "legacy_metadata_identity", "other":
|
|
return r.QueryExceptionCategory
|
|
}
|
|
|
|
message := r.Error
|
|
if r.Failure != nil {
|
|
message = r.Failure.Message + "\n" + message
|
|
}
|
|
switch {
|
|
case strings.Contains(message, "采集 meta 文件身份冲突"):
|
|
return "metadata_identity_conflict"
|
|
case strings.Contains(message, "采集 meta 缺少对应相机、part 的分段记录"):
|
|
return "metadata_pending"
|
|
case strings.Contains(message, "PTS") && (strings.Contains(message, "无效") || strings.Contains(message, "未严格递增")):
|
|
return "source_pts"
|
|
case strings.Contains(message, "meta 与 PTS"):
|
|
return "metadata_pts_mismatch"
|
|
case strings.Contains(message, "PreconditionFailed"):
|
|
return "source_changed"
|
|
case strings.Contains(message, "采集 meta 帧数记录匹配失败") || strings.Contains(message, "采集 meta 帧数记录不唯一") || strings.Contains(message, "采集 meta 帧数记录无效"):
|
|
return "metadata_identity"
|
|
case strings.Contains(message, "采集 meta 缺少对应相机、part"):
|
|
return "legacy_metadata_identity"
|
|
default:
|
|
return "other"
|
|
}
|
|
}
|
|
|
|
func sourceKey(c domain.CaptureRecord) string {
|
|
prefix := c.SourcePrefix
|
|
if prefix == "" {
|
|
prefix = c.Prefix
|
|
}
|
|
// Keep storage namespace and full directory; never deduplicate by display name.
|
|
if prefix == "" {
|
|
prefix = "missing:" + c.ID
|
|
}
|
|
return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join([]string{c.ProjectID, c.ConnectionID, c.Bucket, strings.TrimRight(prefix, "/")}, "\x00"))))
|
|
}
|
|
func partKey(c domain.CaptureRecord) string { c.SourcePrefix = c.Prefix; return sourceKey(c) }
|
|
func mergeStage(a, b string) string {
|
|
if a == "unknown" || b == "unknown" {
|
|
return "unknown"
|
|
}
|
|
if rank(b) < rank(a) {
|
|
return b
|
|
}
|
|
return a
|
|
}
|
|
func stageForItem(i domain.HumanItemRecord, batches map[string]domain.HumanBatchRecord) string {
|
|
if i.Status == domain.HumanItemCancelled {
|
|
return "ended"
|
|
}
|
|
if i.Status == domain.HumanItemAccepted {
|
|
return "delivery"
|
|
}
|
|
if i.PendingCombinedSubmissionID != "" || (i.ProcessingJob != nil && (i.ProcessingJob.Status == "queued" || i.ProcessingJob.Status == "running" || i.ProcessingJob.Status == "failed")) {
|
|
return "annotation"
|
|
}
|
|
for _, s := range batches[i.BatchID].Stages {
|
|
if s.ID == i.CurrentStageID {
|
|
switch s.Kind {
|
|
case domain.HumanStageAnnotation:
|
|
return "annotation"
|
|
case domain.HumanStageReview:
|
|
return "quality"
|
|
case domain.HumanStageAcceptance:
|
|
return "acceptance"
|
|
}
|
|
}
|
|
}
|
|
switch i.CurrentStageID {
|
|
case "annotation_combined", "split_annotation", "cut_adjustment", "semantic_annotation":
|
|
return "annotation"
|
|
case "quality_review", "cut_review", "semantic_review":
|
|
return "quality"
|
|
case "acceptance", "final_acceptance", "prepackage_acceptance":
|
|
return "acceptance"
|
|
}
|
|
return "unknown"
|
|
}
|
|
func rank(s string) int {
|
|
for n, x := range Stages {
|
|
if x == s {
|
|
return n
|
|
}
|
|
}
|
|
return 7
|
|
}
|
|
|
|
func Build(d Snapshot) Report {
|
|
result := Report{Sources: []Source{}, Events: []Event{}}
|
|
captures := map[string][]domain.CaptureRecord{}
|
|
parents := map[string]string{}
|
|
parentNames := map[string]string{}
|
|
captureSource := map[string]string{}
|
|
runSource := map[string]string{}
|
|
runs := map[string]domain.RunRecord{}
|
|
batches := map[string]domain.HumanBatchRecord{}
|
|
for _, r := range d.Runs {
|
|
runs[r.ID] = r
|
|
}
|
|
for _, b := range d.Batches {
|
|
batches[b.ID] = b
|
|
}
|
|
for _, c := range d.Captures {
|
|
key := partKey(c)
|
|
parents[key] = sourceKey(c)
|
|
name := c.CaptureID
|
|
if c.SourcePrefix != "" && strings.TrimRight(c.SourcePrefix, "/") != strings.TrimRight(c.Prefix, "/") {
|
|
name = path.Base(strings.TrimRight(c.SourcePrefix, "/"))
|
|
}
|
|
parentNames[sourceKey(c)] = name
|
|
captures[key] = append(captures[key], c)
|
|
captureSource[c.ID] = key
|
|
if c.RunID != "" {
|
|
runSource[c.RunID] = key
|
|
}
|
|
}
|
|
for _, r := range d.Runs {
|
|
if key := captureSource[r.TaskID]; key != "" {
|
|
runSource[r.ID] = key
|
|
}
|
|
}
|
|
// Human-release packaging children belong to their parent's attempt. Retries don't.
|
|
for changed := true; changed; {
|
|
changed = false
|
|
for _, r := range d.Runs {
|
|
if runSource[r.ID] == "" && r.Trigger == domain.RunTriggerHumanRelease && runSource[r.ParentRunID] != "" {
|
|
runSource[r.ID] = runSource[r.ParentRunID]
|
|
changed = true
|
|
}
|
|
}
|
|
}
|
|
root := func(id string) string {
|
|
seen := map[string]bool{}
|
|
for !seen[id] {
|
|
seen[id] = true
|
|
r, ok := runs[id]
|
|
if !ok || r.Trigger != domain.RunTriggerHumanRelease || r.ParentRunID == "" {
|
|
break
|
|
}
|
|
id = r.ParentRunID
|
|
}
|
|
return id
|
|
}
|
|
latest := map[string]domain.RunRecord{}
|
|
for _, r := range d.Runs {
|
|
key := runSource[r.ID]
|
|
if key == "" || r.Mode == domain.RunModeDebug || r.Trigger == domain.RunTriggerHumanRelease {
|
|
continue
|
|
}
|
|
old := latest[key]
|
|
if old.ID == "" || later(r.CreatedAt, old.CreatedAt) || (r.CreatedAt == old.CreatedAt && r.ID > old.ID) {
|
|
latest[key] = r
|
|
}
|
|
}
|
|
itemSource := map[string]string{}
|
|
taskSource := map[string]string{}
|
|
for _, i := range d.Items {
|
|
if runs[i.RunID].Mode != domain.RunModeDebug {
|
|
itemSource[i.ID] = runSource[i.RunID]
|
|
if i.LegacyTaskID != "" {
|
|
taskSource[i.LegacyTaskID] = runSource[i.RunID]
|
|
}
|
|
}
|
|
}
|
|
for _, t := range d.Tasks {
|
|
if runs[t.RunID].Mode != domain.RunModeDebug {
|
|
taskSource[t.ID] = runSource[t.RunID]
|
|
}
|
|
}
|
|
attemptItems := map[string][]domain.HumanItemRecord{}
|
|
attemptTasks := map[string][]domain.AnnotationTaskRecord{}
|
|
represented := map[string]bool{}
|
|
for _, item := range d.Items {
|
|
attemptItems[root(item.RunID)] = append(attemptItems[root(item.RunID)], item)
|
|
if item.LegacyTaskID != "" {
|
|
represented[item.LegacyTaskID] = true
|
|
}
|
|
}
|
|
for _, task := range d.Tasks {
|
|
attemptTasks[root(task.RunID)] = append(attemptTasks[root(task.RunID)], task)
|
|
}
|
|
itemPartitions := map[string]string{}
|
|
for _, item := range d.Items {
|
|
itemPartitions[item.ID] = item.ReleasePartitionID
|
|
}
|
|
supersededAcceptance := map[string]bool{}
|
|
for _, p := range d.Packages {
|
|
if supersededPackage(p, itemPartitions) {
|
|
supersededAcceptance[p.AcceptanceHumanItemID] = true
|
|
}
|
|
}
|
|
attemptPackages := map[string][]domain.DeliveryPackageRecord{}
|
|
for _, p := range d.Packages {
|
|
rid := root(p.ParentRunID)
|
|
if rid == "" {
|
|
rid = root(p.ChildRunID)
|
|
}
|
|
attemptPackages[rid] = append(attemptPackages[rid], p)
|
|
}
|
|
for key, cs := range captures {
|
|
c := cs[0]
|
|
r := latest[key]
|
|
s := Source{ID: key, ProjectID: c.ProjectID, Name: c.CaptureID, RunID: r.ID, SupplierID: Unassigned, Stage: "intake", Detail: string(c.Status), Tasks: []Task{}}
|
|
for _, candidate := range cs {
|
|
if candidate.DiscoveredAt != "" && (s.DiscoveredAt == "" || later(s.DiscoveredAt, candidate.DiscoveredAt)) {
|
|
s.DiscoveredAt = candidate.DiscoveredAt
|
|
}
|
|
if root(candidate.RunID) == r.ID {
|
|
c = candidate
|
|
}
|
|
}
|
|
if s.Name == "" {
|
|
s.Name = c.Prefix
|
|
}
|
|
s.Detail = string(c.Status)
|
|
if r.ID == "" && c.RunID != "" && runs[c.RunID].Mode != domain.RunModeDebug {
|
|
s.Stage = "unknown"
|
|
s.Detail = "缺少正式处理记录"
|
|
}
|
|
s.WorkflowID = r.WorkflowID
|
|
s.Runs = []RunReference{}
|
|
if r.ID != "" {
|
|
s.Runs = append(s.Runs, RunReference{r.ID, r.WorkflowID, c.CaptureID})
|
|
}
|
|
suppliers := map[string]bool{}
|
|
hasTasks := false
|
|
minStage := "ended"
|
|
for _, i := range attemptItems[r.ID] {
|
|
if supersededAcceptance[i.ID] {
|
|
continue
|
|
}
|
|
if itemSource[i.ID] != key || root(i.RunID) != r.ID || r.ID == "" {
|
|
continue
|
|
}
|
|
hasTasks = true
|
|
stage := stageForItem(i, batches)
|
|
s.Tasks = append(s.Tasks, Task{i.ID, stage, string(i.Status)})
|
|
minStage = mergeStage(minStage, stage)
|
|
if batches[i.BatchID].SourceBatchID == "" {
|
|
supplier := i.SupplierID
|
|
if supplier == "" {
|
|
supplier = Unassigned
|
|
}
|
|
suppliers[supplier] = true
|
|
}
|
|
if i.ProcessingJob != nil && i.ProcessingJob.Status == "failed" {
|
|
s.Exception = true
|
|
}
|
|
}
|
|
|
|
for _, task := range attemptTasks[r.ID] {
|
|
if represented[task.ID] || taskSource[task.ID] != key || root(task.RunID) != r.ID || r.ID == "" {
|
|
continue
|
|
}
|
|
stage := "unknown"
|
|
switch task.Status {
|
|
case domain.AnnotationTaskStatusPendingAnnotation, domain.AnnotationTaskStatusRework:
|
|
stage = "annotation"
|
|
case domain.AnnotationTaskStatusPendingQuality:
|
|
stage = "quality"
|
|
case domain.AnnotationTaskStatusPendingAcceptance:
|
|
stage = "acceptance"
|
|
case domain.AnnotationTaskStatusAccepted:
|
|
stage = "delivery"
|
|
}
|
|
hasTasks = true
|
|
s.Tasks = append(s.Tasks, Task{task.ID, stage, string(task.Status)})
|
|
minStage = mergeStage(minStage, stage)
|
|
supplier := task.SupplierID
|
|
if supplier == "" {
|
|
supplier = Unassigned
|
|
}
|
|
suppliers[supplier] = true
|
|
}
|
|
if len(suppliers) == 1 {
|
|
for supplier := range suppliers {
|
|
s.SupplierID = supplier
|
|
}
|
|
} else if len(suppliers) > 1 || (hasTasks && len(suppliers) == 0) {
|
|
s.SupplierID = Unknown
|
|
}
|
|
if hasTasks {
|
|
s.Stage = minStage
|
|
s.Detail = minStage
|
|
} else if r.ID != "" {
|
|
s.Detail = string(r.Status)
|
|
switch r.Status {
|
|
case domain.RunStatusCompleted:
|
|
s.Stage = "unknown"
|
|
case domain.RunStatusCancelled:
|
|
s.Stage = "ended"
|
|
}
|
|
}
|
|
if root(c.RunID) == r.ID || r.ID == "" {
|
|
switch c.BusinessStage {
|
|
case domain.CaptureStageDelivered:
|
|
if !hasTasks || minStage == "delivery" {
|
|
s.Stage = "delivered"
|
|
}
|
|
case domain.CaptureStageNoDeliverables, domain.CaptureStageDiscarded, domain.CaptureStageCancelled:
|
|
s.Stage = "ended"
|
|
}
|
|
}
|
|
|
|
packageStage := "ended"
|
|
packageCount := 0
|
|
for _, p := range attemptPackages[r.ID] {
|
|
if p.Mode == domain.RunModeDebug || p.Temporary || r.ID == "" || (root(p.ParentRunID) != r.ID && root(p.ChildRunID) != r.ID) {
|
|
continue
|
|
}
|
|
if supersededPackage(p, itemPartitions) {
|
|
continue
|
|
}
|
|
packageCount++
|
|
stage := "unknown"
|
|
switch p.Status {
|
|
case domain.DeliveryPackagePendingAcceptance, domain.DeliveryPackageRejected:
|
|
stage = "acceptance"
|
|
case domain.DeliveryPackageBuilding, domain.DeliveryPackageFailed:
|
|
stage = "delivery"
|
|
case domain.DeliveryPackageAccepted:
|
|
stage = "delivered"
|
|
case domain.DeliveryPackageNoDeliverables:
|
|
stage = "ended"
|
|
}
|
|
if p.Status == domain.DeliveryPackageFailed {
|
|
s.Exception = true
|
|
}
|
|
packageStage = mergeStage(packageStage, stage)
|
|
}
|
|
if packageCount > 0 && (!hasTasks || minStage == "delivery") {
|
|
s.Stage = packageStage
|
|
s.Detail = packageStage
|
|
}
|
|
if (root(c.RunID) == r.ID || c.RunID == "") && (c.Status == domain.CaptureRecordDiscarded || c.Status == domain.CaptureRecordCancelled) {
|
|
s.Stage = "ended"
|
|
s.Detail = string(c.Status)
|
|
}
|
|
if r.DeliveryOutcome == "no_deliverables" {
|
|
s.Stage = "ended"
|
|
s.Detail = "no_deliverables"
|
|
}
|
|
if r.Status == domain.RunStatusFailed || (root(c.RunID) == r.ID && c.Status == domain.CaptureRecordFailed) {
|
|
s.Exception = true
|
|
}
|
|
if s.Exception {
|
|
s.ExceptionCategory = exceptionCategory(r)
|
|
}
|
|
if s.SupplierID == Unknown || s.Stage == "unknown" {
|
|
result.Incomplete = true
|
|
}
|
|
sort.Slice(s.Tasks, func(i, j int) bool { return s.Tasks[i].ID < s.Tasks[j].ID })
|
|
result.Sources = append(result.Sources, s)
|
|
}
|
|
sort.Slice(result.Sources, func(i, j int) bool { return result.Sources[i].ID < result.Sources[j].ID })
|
|
buildEvents(d, &result, runSource, itemSource, taskSource, runs)
|
|
for n := range result.Events {
|
|
result.Events[n].AttemptID = root(result.Events[n].RunID)
|
|
}
|
|
for _, run := range d.Runs {
|
|
if run.Mode != domain.RunModeDebug && run.Trigger != domain.RunTriggerHumanRelease && runSource[run.ID] != "" {
|
|
result.Events = append(result.Events, Event{ID: run.ID, SourceID: runSource[run.ID], Metric: "attempt_started", At: run.CreatedAt, AttemptID: run.ID})
|
|
}
|
|
}
|
|
// An unadmitted parent may be replaced by discovered part directories.
|
|
// Preserve its registration date, but it is no longer required production work.
|
|
hasParts := map[string]bool{}
|
|
for part, parent := range parents {
|
|
if part != parent {
|
|
hasParts[parent] = true
|
|
}
|
|
}
|
|
placeholders := map[string]bool{}
|
|
for key, cs := range captures {
|
|
if !hasParts[key] || latest[key].ID != "" {
|
|
continue
|
|
}
|
|
unadmitted := true
|
|
for _, c := range cs {
|
|
if c.RunID != "" || c.Status == domain.CaptureRecordDiscarded || c.Status == domain.CaptureRecordLaunching {
|
|
unadmitted = false
|
|
}
|
|
}
|
|
placeholders[key] = unadmitted
|
|
}
|
|
return rollupParts(result, parents, parentNames, placeholders, requiredMilestoneUnits(d, result.Events, root))
|
|
}
|
|
|
|
func later(a, b string) bool {
|
|
left, e1 := time.Parse(time.RFC3339Nano, a)
|
|
right, e2 := time.Parse(time.RFC3339Nano, b)
|
|
return e1 == nil && (e2 != nil || left.After(right))
|
|
}
|
|
|
|
// A physical Capture may register several part directories. Retries are resolved
|
|
// per part first; completion of one part must not hide unfinished sibling work.
|
|
func rollupParts(parts Report, parents, names map[string]string, placeholders map[string]bool, units milestoneUnits) Report {
|
|
out := Report{Sources: []Source{}, Events: parts.Events, Incomplete: parts.Incomplete}
|
|
combined := map[string]*Source{}
|
|
discovery := map[string]string{}
|
|
for _, part := range parts.Sources {
|
|
id := parents[part.ID]
|
|
if part.DiscoveredAt != "" && (discovery[id] == "" || later(discovery[id], part.DiscoveredAt)) {
|
|
discovery[id] = part.DiscoveredAt
|
|
}
|
|
if placeholders[part.ID] {
|
|
continue
|
|
}
|
|
s := combined[id]
|
|
if s == nil {
|
|
copy := part
|
|
copy.ID = id
|
|
if names[id] != "" {
|
|
copy.Name = names[id]
|
|
}
|
|
combined[id] = ©
|
|
continue
|
|
}
|
|
s.Stage = mergeStage(s.Stage, part.Stage)
|
|
s.Detail = s.Stage
|
|
s.Exception = s.Exception || part.Exception
|
|
if s.SupplierID != part.SupplierID {
|
|
s.SupplierID = Unknown
|
|
out.Incomplete = true
|
|
}
|
|
if s.DiscoveredAt == "" || later(s.DiscoveredAt, part.DiscoveredAt) {
|
|
s.DiscoveredAt = part.DiscoveredAt
|
|
}
|
|
s.Tasks = append(s.Tasks, part.Tasks...)
|
|
s.Runs = append(s.Runs, part.Runs...)
|
|
s.RunID = ""
|
|
s.WorkflowID = ""
|
|
}
|
|
for _, s := range combined {
|
|
s.DiscoveredAt = discovery[s.ID]
|
|
out.Sources = append(out.Sources, *s)
|
|
}
|
|
sort.Slice(out.Sources, func(i, j int) bool { return out.Sources[i].ID < out.Sources[j].ID })
|
|
out.Events = sourceMilestones(parts.Events, parents, placeholders, units)
|
|
for _, s := range out.Sources {
|
|
if s.DiscoveredAt != "" {
|
|
out.Events = append(out.Events, Event{ID: "new:" + s.ID, SourceID: s.ID, SupplierID: Unassigned, Metric: "new", At: s.DiscoveredAt})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Rework preserves the old package for audit but moves its production items to
|
|
// a new partition. Only explicit current pointers justify excluding it.
|
|
func supersededPackage(p domain.DeliveryPackageRecord, itemPartitions map[string]string) bool {
|
|
if p.ReleasePartitionID == "" || len(p.ItemIDs) == 0 {
|
|
return false
|
|
}
|
|
for _, id := range p.ItemIDs {
|
|
if itemPartitions[id] == "" || itemPartitions[id] == p.ReleasePartitionID {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|