From b2324c19a657fb1339ab635f2b5e5ee99d46f359 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 29 Oct 2026 10:43:15 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Bound=20QC=20candidates=20by=20acce?= =?UTF-8?q?ss=20scope=20and=20read=20only=20priced=20slice=20versions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .scratch/poll-pool-optimization/work.md | 11 ++ docs/workforce-statistics.zh-CN.md | 2 +- internal/api/human_loop.go | 33 ++++ internal/api/human_pool_project_scope_test.go | 144 ++++++++++++++++++ internal/domain/human_loop.go | 3 + internal/store/human_loop.go | 15 +- internal/store/human_pool_read_scope_test.go | 70 +++++++++ .../store/workforce_slice_history_test.go | 51 +++++++ internal/store/workforce_stats.go | 66 ++++---- internal/workforcestats/slice_versions.go | 66 ++++++++ .../workforcestats/slice_versions_test.go | 51 +++++++ 11 files changed, 480 insertions(+), 32 deletions(-) create mode 100644 internal/api/human_pool_project_scope_test.go create mode 100644 internal/store/human_pool_read_scope_test.go create mode 100644 internal/store/workforce_slice_history_test.go create mode 100644 internal/workforcestats/slice_versions.go create mode 100644 internal/workforcestats/slice_versions_test.go diff --git a/.scratch/poll-pool-optimization/work.md b/.scratch/poll-pool-optimization/work.md index cd79b100..23836adb 100644 --- a/.scratch/poll-pool-optimization/work.md +++ b/.scratch/poll-pool-optimization/work.md @@ -115,3 +115,14 @@ Risk: mixed-version response compatibility, permissions, in-flight workflow reco - Baseline round one completed 1,800 steady seconds with 48,750 requests, zero HTTP errors and zero Mongo writes. Available-pool p95 ranges 491.447–822.886 ms. Stage-one round one is running; no candidate long-run comparison is available yet. This does not resolve the original unclassified warmup failures. - Closed an analysis-only evidence gap: loading reports now reuses the runner's strict revision/mode/fixture/timing/request/resource validation before computing acceptance metrics. Tests reject reports from another commit or configuration and missing resource evidence. All eight Python runner/analyzer tests passed. No application or active load harness changes. + +## 2026-10-29 post-release 50 Mbps latency diagnosis and repair + +- Public HTTP read probes reproduced workforce waiting before first byte: QC 2.50–2.85s/~550 compressed bytes; admin 15.97–18.90s/~8.8KB; annotator 4.31s then 28ms. Download about 2–3ms. This identifies response-preparation delay, not egress-body saturation for these calls. API pods sampled at 3–10m CPU/51–188MiB; a snapshot is not throttling history. +- Corrected probe fidelity: initial QC available probes omitted frontend pluginId and timed out at 30s. Those do not represent the actual page request. Complete frontend pool scope (annotation-qa, dual QC stages, limit 50) returned in 6.72s; mine in 0.163s. Actual default/trace and reviewed calls can differ; do not claim every task request is measured. +- Read-only Mongo plans: existing sorted pool indexes are present. Complete plugin-scoped candidate plan scanned/returned 100 documents in 618ms; omitted-plugin diagnostic scanned 3,939 with blocking sort (~10s). Do not prescribe a missing pool index from that broader query. +- Confirmed pool waste: API fetches up to six raw pages before permission filtering; other-project/supplier rows can exhaust that cap and leave visible tasks after its cursor. Added server-only QC project/supplier necessary-condition filters for annotation-qa available pool. Membership scope is request-local, legacy blank supplier remains, shared acceptance paths are excluded, and final role/supplier/self-review checks remain. No cross-request permission cache. Multi-page traversal regression checks final visible tasks, even though fewer invisible rows changes where a page boundary falls. +- Read-only execution of the repaired GET handler against current data returned 17 tasks/119,034 raw response bytes in 735ms. This is local handler plus Mongo forwarding, not deployed public HTTP acceptance. The old first page could be empty while hasMore=true; do not claim equal first-page counts on that production sample. +- Workforce loaded every slice revision of joined tasks. New calculator-derived version set applies canonical submission deduplication, skips valid frozen durations, and retains legacy quality-approved-version acceptance fallback. Mongo reads these task/version pairs in bounded groups of 200. A local 1,000-extra-history fixture reads one relevant version with unchanged output; matrix preserves Memory/Mongo reports for author/QC/acceptance/org and details scopes. +- Real-data read-only stats diagnostic completed with 357 required slice versions. Earlier broad read exceeded its 120s diagnostic context. The forwarding channel transfers slowly; its ~99s completed time is not public HTTP timing and cannot establish a dashboard latency percentage. Submission, attribution/revision/task/run history still incurs reads; reviewOperations currently has only _id_ index and showed ~2s initial query. Further index work, if needed, requires a separately reviewed production plan; none created here. +- Targeted tests and local isolated Mongo parity passed; full Go gate: 68 failures all in existing baseline (70 known), no new failures. Race-enabled added scope/statistics tests passed, including isolated Mongo and >six-round cursor traversal. No production mutations, index creation, push, PR update or deployment for these repairs. diff --git a/docs/workforce-statistics.zh-CN.md b/docs/workforce-statistics.zh-CN.md index 73202d68..d4536fdf 100644 --- a/docs/workforce-statistics.zh-CN.md +++ b/docs/workforce-statistics.zh-CN.md @@ -39,7 +39,7 @@ 新送检工作记录增加稳定提交 ID、首交/返工标记、视频时长;同一提交的工作记录使用固定 ID 并仅首次写入,重试保留原来的成功时间和归属。读取旧事件时批量关联相关任务的提交历史,同一任务、同一正式版本按最早成功事件去重,避免跨日期重试再次入账。 -查询使用区间和组织/供应商/项目过滤、字段投影与批量关联,正文仅在展开明细时加载。本人范围只关联本人决定涉及的任务;切片区间只为送检提交、本人质检决定和终审来源读取,不随打回归属关联扩大。未增加线上索引或持久化统计表。 +查询使用区间和组织/供应商/项目过滤、字段投影与批量关联,正文仅在展开明细时加载。本人范围只关联本人决定涉及的任务;切片区间只为送检提交、本人质检决定和终审来源读取,不随打回归属关联扩大。未增加线上索引或持久化统计表。 切片区间按计算器会使用的“任务+正式版本”分批读取:有可靠冻结时长时不再读取对应切片;旧提交按既有去重规则选择版本,历史终审仍按决定前最后一次质检通过的版本回退。不会为期间关联到的任务加载全部自动保存历史。 ## 数据边界与回退 diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index a8d02d13..1c1507b0 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -366,6 +366,39 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht if randomClaimAnnotator(principal) && query.ActorUserID == "" && (query.PluginID == "" || query.PluginID == "annotation-qa") { query.BlindAnnotationActorID = principal.User.ID } + // QC always needs a membership in the item's project. Restrict candidate + // reads to this necessary scope before pagination; supplier/stage/read/claim + // checks remain authoritative below. Shared acceptance is deliberately excluded. + if pool == "available" && query.PluginID == "annotation-qa" && principal != nil && principal.User.Role == domain.UserRoleQualityReviewer { + ctx, err := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, nil) + if err != nil { + renderError(writer, 500, err) + return nil, request + } + request = request.WithContext(ctx) + access := server.acceptanceAccessFor(ctx, principal.User.ID) + if access.err != nil { + renderError(writer, 500, access.err) + return nil, request + } + if access.user != nil && access.user.Role == domain.UserRoleQualityReviewer { + roles, err := server.annotationMembershipsForUser(ctx, server.requestOrganizationID(request), "", principal.User.ID) + if err != nil { + renderError(writer, 500, err) + return nil, request + } + query.AvailableSupplierID = &access.user.SupplierID + query.AvailableProjectIDs = []string{} + seen := map[string]bool{} + for _, role := range roles { + if !seen[role.ProjectID] { + seen[role.ProjectID] = true + query.AvailableProjectIDs = append(query.AvailableProjectIDs, role.ProjectID) + } + } + sort.Strings(query.AvailableProjectIDs) + } + } // List responses never need the workbench draft or disposition histories. query.ListProjection = true fetchSize := limit diff --git a/internal/api/human_pool_project_scope_test.go b/internal/api/human_pool_project_scope_test.go new file mode 100644 index 00000000..31676d1f --- /dev/null +++ b/internal/api/human_pool_project_scope_test.go @@ -0,0 +1,144 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/store" + "encoding/json" + "fmt" + "net/http" + "testing" +) + +type qcProjectScopeProbe struct { + store.Store + legacy bool + candidates int +} + +func (p *qcProjectScopeProbe) ListHumanItems(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) { + if p.legacy { + q.AvailableProjectIDs = nil + q.AvailableSupplierID = nil + } + rows, err := p.Store.ListHumanItems(ctx, org, q) + if q.PoolCandidateProjection { + p.candidates += len(rows) + } + return rows, err +} +func TestHumanPoolQCProjectScopeSkipsInvisibleBacklog(t *testing.T) { + f := newCombinedDraftFixture(t) + item := f.item + item.CurrentStageID = "quality_review" + item.AssignedUserID = "" + item.Status = domain.HumanItemReady + item.SupplierID = f.reviewer.SupplierID + if err := f.store.UpsertHumanItem(f.ctx, item); err != nil { + t.Fatal(err) + } + for i := 0; i < 90; i++ { + noise := item + noise.ID = fmt.Sprintf("hidden-%03d", i) + if i%2 == 0 { + noise.ProjectID = "foreign-project" + } else { + noise.SupplierID = "foreign-supplier" + } + if err := f.store.UpsertHumanItem(f.ctx, noise); err != nil { + t.Fatal(err) + } + } + path := "/api/human-items?pluginId=annotation-qa&pool=available&stageIds=quality_review,semantic_review&limit=50" + old := &qcProjectScopeProbe{Store: f.store, legacy: true} + f.handler = NewHandler(old, nil) + before := f.request(http.MethodGet, path, f.reviewer, nil) + probe := &qcProjectScopeProbe{Store: f.store} + f.handler = NewHandler(probe, nil) + after := f.request(http.MethodGet, path, f.reviewer, nil) + if before.Code != 200 || after.Code != 200 || before.Body.String() != after.Body.String() { + t.Fatalf("changed response %d/%d", before.Code, after.Code) + } + var page struct{ Items []domain.HumanItemRecord } + if err := json.Unmarshal(after.Body.Bytes(), &page); err != nil || len(page.Items) != 1 { + t.Fatalf("expected one visible task: %s", after.Body.String()) + } + if old.candidates != 91 || probe.candidates != 1 { + t.Fatalf("candidate reads %d -> %d", old.candidates, probe.candidates) + } + // Permissions are rebuilt on each request. A disabled account cannot reuse scope. + user := f.reviewer + user.Status = domain.UserStatusDisabled + if err := f.store.UpsertUser(f.ctx, user); err != nil { + t.Fatal(err) + } + after = f.request(http.MethodGet, path, user, nil) + if after.Code == 200 { + var p struct{ Items []domain.HumanItemRecord } + json.Unmarshal(after.Body.Bytes(), &p) + if len(p.Items) > 0 { + t.Fatal("disabled account retained visible tasks") + } + } +} + +func TestHumanPoolQCProjectScopePreservesMultiPageVisibleTasks(t *testing.T) { + f := newCombinedDraftFixture(t) + item := f.item + item.CurrentStageID = "quality_review" + item.AssignedUserID = "" + item.Status = domain.HumanItemReady + item.SupplierID = f.reviewer.SupplierID + item.CreatedAt = "2026-01-01T00:00:00Z" + if err := f.store.UpsertHumanItem(f.ctx, item); err != nil { + t.Fatal(err) + } + for i := 0; i < 700; i++ { + noise := item + noise.ID = fmt.Sprintf("noise-%04d", i) + noise.SupplierID = "foreign-supplier" + noise.CreatedAt = "2026-02-01T00:00:00Z" + if err := f.store.UpsertHumanItem(f.ctx, noise); err != nil { + t.Fatal(err) + } + } + collect := func(legacy bool) []string { + f.handler = NewHandler(&qcProjectScopeProbe{Store: f.store, legacy: legacy}, nil) + cursor := "" + ids := []string{} + for page := 0; page < 10; page++ { + path := "/api/human-items?pluginId=annotation-qa&pool=available&stageIds=quality_review,semantic_review&limit=5" + if cursor != "" { + path += "&cursor=" + cursor + } + r := f.request(http.MethodGet, path, f.reviewer, nil) + if r.Code != 200 { + t.Fatalf("list failed %d", r.Code) + } + var out struct { + Items []domain.HumanItemRecord + HasMore bool + NextCursor string + } + if err := json.Unmarshal(r.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + for _, it := range out.Items { + ids = append(ids, it.ID) + } + if !out.HasMore { + return ids + } + if out.NextCursor == "" || out.NextCursor == cursor { + t.Fatal("cursor failed to advance") + } + cursor = out.NextCursor + } + t.Fatal("pagination did not finish") + return nil + } + before, after := collect(true), collect(false) + if len(before) != 1 || len(after) != 1 || before[0] != after[0] { + t.Fatalf("visible tasks changed %v %v", before, after) + } +} diff --git a/internal/domain/human_loop.go b/internal/domain/human_loop.go index 554519cf..21f2a64c 100644 --- a/internal/domain/human_loop.go +++ b/internal/domain/human_loop.go @@ -326,6 +326,9 @@ type HumanItemCursor struct { } type HumanItemQuery struct { + // Server-derived necessary QC project scope; nil is unrestricted, empty matches nothing. + AvailableProjectIDs []string + AvailableSupplierID *string // Server-populated review history scope; nil means unrestricted. IDs []string LegacyTaskID string diff --git a/internal/store/human_loop.go b/internal/store/human_loop.go index b3a5bc28..77ec21db 100644 --- a/internal/store/human_loop.go +++ b/internal/store/human_loop.go @@ -45,7 +45,7 @@ func matchesHumanMinePool(item domain.HumanItemRecord, query domain.HumanItemQue } func humanItemMatches(item domain.HumanItemRecord, query domain.HumanItemQuery) bool { - return matchesHumanMinePool(item, query) && matchesAnnotationView(item, query) && (query.IDs == nil || containsHumanItemID(query.IDs, item.ID)) && (query.LegacyTaskID == "" || item.LegacyTaskID == query.LegacyTaskID) && + return (query.AvailableSupplierID == nil || item.SupplierID == "" || item.SupplierID == *query.AvailableSupplierID) && (query.AvailableProjectIDs == nil || containsHumanItemID(query.AvailableProjectIDs, item.ProjectID)) && matchesHumanMinePool(item, query) && matchesAnnotationView(item, query) && (query.IDs == nil || containsHumanItemID(query.IDs, item.ID)) && (query.LegacyTaskID == "" || item.LegacyTaskID == query.LegacyTaskID) && (query.ActorUserID == "" || item.AssignedUserID == query.ActorUserID || item.LatestAuthorUserID == query.ActorUserID) && (query.PluginID == "" || item.PluginID == query.PluginID) && (query.BatchID == "" || item.BatchID == query.BatchID) && @@ -615,6 +615,13 @@ func humanBatchMongoFilter(organizationID string, query domain.HumanBatchQuery) func humanItemMongoFilter(organizationID string, query domain.HumanItemQuery) bson.M { filter := bson.M{"organizationId": normalizedOrganizationID(organizationID)} + if query.AvailableSupplierID != nil { + filter["supplierId"] = bson.M{"$in": bson.A{"", nil, *query.AvailableSupplierID}} + } + if query.AvailableProjectIDs != nil { + filter["projectId"] = bson.M{"$in": query.AvailableProjectIDs} + } + if query.IDs != nil { filter["id"] = bson.M{"$in": query.IDs} } @@ -634,7 +641,11 @@ func humanItemMongoFilter(organizationID string, query domain.HumanItemQuery) bs filter["batchId"] = query.BatchID } if query.ProjectID != "" { - filter["projectId"] = query.ProjectID + if query.AvailableProjectIDs == nil || containsHumanItemID(query.AvailableProjectIDs, query.ProjectID) { + filter["projectId"] = query.ProjectID + } else { + filter["projectId"] = bson.M{"$in": []string{}} + } } if query.WorkflowID != "" { filter["workflowId"] = query.WorkflowID diff --git a/internal/store/human_pool_read_scope_test.go b/internal/store/human_pool_read_scope_test.go new file mode 100644 index 00000000..067548f3 --- /dev/null +++ b/internal/store/human_pool_read_scope_test.go @@ -0,0 +1,70 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "reflect" + "testing" + "time" +) + +func TestHumanPoolReadScopeMongoParity(t *testing.T) { + if os.Getenv("TEST_WORKFORCE_MONGO_URI") == "" { + t.Skip("local Mongo opt-in") + } + if os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" { + t.Fatal("loopback only") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cl, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018")) + if err != nil { + t.Fatal(err) + } + defer cl.Disconnect(context.Background()) + db := cl.Database(fmt.Sprintf("qc_scope_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + ms := NewMemoryStore() + ds := NewMongoStore(db) + for i, supplier := range []string{"vendor", "", "other"} { + for j, project := range []string{"p", "other", ""} { + it := domain.HumanItemRecord{ID: fmt.Sprintf("%d-%d", i, j), OrganizationID: "o", SupplierID: supplier, ProjectID: project, CreatedAt: "2026-10-01T00:00:00Z"} + if err := ms.UpsertHumanItem(ctx, it); err != nil { + t.Fatal(err) + } + if err := ds.UpsertHumanItem(ctx, it); err != nil { + t.Fatal(err) + } + } + } + // Missing legacy supplier fields must behave exactly like empty strings. + if _, err := db.Collection("humanItems").UpdateMany(ctx, bson.M{"supplierId": ""}, bson.M{"$unset": bson.M{"supplierId": ""}}); err != nil { + t.Fatal(err) + } + supplier := "vendor" + for _, q := range []domain.HumanItemQuery{{}, {AvailableProjectIDs: []string{}}, {AvailableProjectIDs: []string{"p"}, AvailableSupplierID: &supplier}, {AvailableProjectIDs: []string{"p"}, AvailableSupplierID: &supplier, ProjectID: "other"}, {AvailableProjectIDs: []string{""}, AvailableSupplierID: &supplier}} { + a, e := ms.ListHumanItems(ctx, "o", q) + if e != nil { + t.Fatal(e) + } + b, e := ds.ListHumanItems(ctx, "o", q) + if e != nil { + t.Fatal(e) + } + ids := func(items []domain.HumanItemRecord) []string { + out := []string{} + for _, it := range items { + out = append(out, it.ID) + } + return out + } + if !reflect.DeepEqual(ids(a), ids(b)) { + t.Fatalf("scope mismatch %v %v", ids(a), ids(b)) + } + } +} diff --git a/internal/store/workforce_slice_history_test.go b/internal/store/workforce_slice_history_test.go new file mode 100644 index 00000000..a3373200 --- /dev/null +++ b/internal/store/workforce_slice_history_test.go @@ -0,0 +1,51 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "testing" + "time" +) + +func TestWorkforceStatsReadsOnlyPricedSliceVersions(t *testing.T) { + if os.Getenv("TEST_WORKFORCE_MONGO_URI") == "" { + t.Skip("local Mongo opt-in") + } + if os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" { + t.Fatal("loopback only") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cl, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018")) + if err != nil { + t.Fatal(err) + } + defer cl.Disconnect(context.Background()) + db := cl.Database(fmt.Sprintf("workforce_versions_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + ds := NewMongoStore(db) + seedWorkforceStats(t, ds) + docs := []any{} + for revision := 10; revision < 1010; revision++ { + docs = append(docs, bson.M{"id": fmt.Sprintf("old-%d", revision), "organizationId": "workforce-test", "taskId": "task", "revision": revision, "segments": []domain.SliceSegment{{Start: 0, End: 999}}}) + } + if _, err := db.Collection("sliceRevisions").InsertMany(ctx, docs); err != nil { + t.Fatal(err) + } + d, err := ds.LoadWorkforceStats(ctx, "workforce-test", workforcestats.Query{}) + if err != nil { + t.Fatal(err) + } + if len(d.Slices) != 1 || d.Slices[0].Revision != 4 { + t.Fatalf("read unrelated historical versions: %d", len(d.Slices)) + } + if got := workforcestats.Build(d, workforcestats.Query{}).Totals.FirstVideoSeconds; got != 79 { + t.Fatalf("wrong legacy output seconds %v", got) + } +} diff --git a/internal/store/workforce_stats.go b/internal/store/workforce_stats.go index 2872a9ea..0ca3687f 100644 --- a/internal/store/workforce_stats.go +++ b/internal/store/workforce_stats.go @@ -55,31 +55,6 @@ func workforceTaskIDs(d workforcestats.Snapshot) []string { return out } -// workforceVideoTaskIDs lists tasks whose slice ranges price video seconds: -// in-window submissions, the reviewer's quality decisions and the sources of -// final-acceptance decisions. -func workforceVideoTaskIDs(d workforcestats.Snapshot) []string { - ids := map[string]bool{} - for _, e := range d.Events { - switch e.EventType { - case "annotation_submitted", "combined_submitted", "rework_submitted", "quality_approved", "quality_rejected": - if e.TaskID != "" { - ids[e.TaskID] = true - } - } - } - for _, item := range d.SourceItems { - if item.LegacyTaskID != "" { - ids[item.LegacyTaskID] = true - } - } - out := make([]string, 0, len(ids)) - for id := range ids { - out = append(out, id) - } - return out -} - // workforceAcceptanceSourceIDs lists the source items final-acceptance // decisions were made on. func workforceAcceptanceSourceIDs(ops []domain.ReviewOperation) []string { @@ -572,12 +547,45 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf } // Slice ranges only price submissions and the scope's own review // decisions, never every task joined for return attribution. - if sliceIDs := workforceVideoTaskIDs(d); len(sliceIDs) > 0 { - sliceFilter := organizationScopedFilter(org, bson.M{"taskId": bson.M{"$in": sliceIDs}}) - if err := read(s.sliceRevisions, sliceFilter, fields("taskId", "revision", "segments.id", "segments.start", "segments.end"), &d.Slices); err != nil { - return d, err + // Price only versions the shared calculator can actually consult. + // Reading every historical slice version of these tasks grows with + // autosaves rather than the report's output and can dominate the load. + versions := workforcestats.SliceVersions(d) + clauses := bson.A{} + taskIDs := make([]string, 0, len(versions)) + for id := range versions { + taskIDs = append(taskIDs, id) + } + sort.Strings(taskIDs) + flush := func() error { + if len(clauses) == 0 { + return nil + } + var rows []domain.SliceRevisionRecord + err := read(s.sliceRevisions, organizationScopedFilter(org, bson.M{"$or": clauses}), fields("taskId", "revision", "segments.start", "segments.end"), &rows) + if err == nil { + d.Slices = append(d.Slices, rows...) + } + clauses = nil + return err + } + for _, id := range taskIDs { + revisions := []int{} + for version := range versions[id] { + revisions = append(revisions, version) + } + sort.Ints(revisions) + clauses = append(clauses, bson.M{"taskId": id, "revision": bson.M{"$in": revisions}}) + if len(clauses) == 200 { + if err := flush(); err != nil { + return d, err + } } } + if err := flush(); err != nil { + return d, err + } + if err := read(s.annotationTasks, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": ids}}), fields("id", "projectId", "runId", "supplierId", "templateVersion"), &d.Tasks); err != nil { return d, err } diff --git a/internal/workforcestats/slice_versions.go b/internal/workforcestats/slice_versions.go new file mode 100644 index 00000000..92a53c62 --- /dev/null +++ b/internal/workforcestats/slice_versions.go @@ -0,0 +1,66 @@ +package workforcestats + +import "droboticflow/internal/domain" + +// SliceVersions returns the exact versions Build can use to price video time. +// Frozen event/acceptance seconds need no slice read. Legacy quality and final +// acceptance records retain the reviewed-version lookup and its submission fallback. +func SliceVersions(d Snapshot) map[string]map[int]bool { + result := map[string]map[int]bool{} + add := func(task string, revision int) { + if task != "" { + if result[task] == nil { + result[task] = map[int]bool{} + } + result[task][revision] = true + } + } + // Apply exactly the calculator's canonical duplicate selection. History + // is attribution evidence, not another set of in-window submissions. + canonical := map[string]domain.AnnotationWorklogEventRecord{} + for _, e := range append(append([]domain.AnnotationWorklogEventRecord(nil), d.SubmissionHistory...), d.Events...) { + if !submission(e) { + continue + } + key := submissionKey(e) + old, ok := canonical[key] + if !ok || parsedAfter(old.CreatedAt, e.CreatedAt) || (old.CreatedAt == e.CreatedAt && e.ID < old.ID) { + canonical[key] = e + } + } + for _, e := range d.Events { + if submission(e) { + e = canonical[submissionKey(e)] + } else if e.EventType != "quality_approved" && e.EventType != "quality_rejected" { + continue + } + if !validSeconds(e.VideoSeconds) { + add(e.TaskID, e.Revision) + } + } + + items := map[string]string{} + for _, v := range d.SourceItems { + items[v.ID] = v.LegacyTaskID + } + for _, op := range d.AcceptanceDecisions { + fixed := map[string]domain.ReviewVideoSource{} + for _, v := range op.ReviewVideoSources { + fixed[v.SourceItemID] = v + } + for _, id := range op.SourceItemIDs { + v, frozen := fixed[id] + if frozen && validSeconds(v.VideoSeconds) { + continue + } + task := items[id] + if frozen && v.TaskID != "" { + task = v.TaskID + } + if revision, ok := LatestQualityApprovedRevision(d.Reviews, task, op.CreatedAt); ok { + add(task, revision) + } + } + } + return result +} diff --git a/internal/workforcestats/slice_versions_test.go b/internal/workforcestats/slice_versions_test.go new file mode 100644 index 00000000..eea3868f --- /dev/null +++ b/internal/workforcestats/slice_versions_test.go @@ -0,0 +1,51 @@ +package workforcestats + +import ( + "droboticflow/internal/domain" + "math" + "reflect" + "testing" +) + +func TestSliceVersionsPreservesReportAndExcludesAutosaves(t *testing.T) { + frozen := 12.0 + invalid := math.NaN() + d := Snapshot{ + Events: []domain.AnnotationWorklogEventRecord{ + {ID: "submit", TaskID: "task", Revision: 2, EventType: "combined_submitted", UserID: "author", CreatedAt: at}, + {ID: "qc", TaskID: "task", Revision: 3, EventType: "quality_approved", UserID: "qc", CreatedAt: at, VideoSeconds: &invalid}, + {ID: "frozen", TaskID: "frozen", Revision: 1, EventType: "combined_submitted", UserID: "author", CreatedAt: at, VideoSeconds: &frozen}, + }, + Reviews: []domain.AnnotationReviewRecord{{TaskID: "legacy", Revision: 7, Stage: domain.AnnotationReviewStageQuality, Decision: domain.AnnotationReviewApproved, CreatedAt: "2026-09-27T00:00:00Z"}}, + SourceItems: []domain.HumanItemRecord{{ID: "source", LegacyTaskID: "legacy"}}, + AcceptanceDecisions: []domain.ReviewOperation{{ID: "accept", Status: "completed", Decision: "approved", StageID: "acceptance", UserID: "acceptor", CreatedAt: at, CompletedAt: at, SourceItemIDs: []string{"source"}}}, + } + // Most historical versions cannot influence this report. + for _, task := range []string{"task", "frozen", "legacy", "unrelated"} { + for revision := 1; revision <= 100; revision++ { + d.Slices = append(d.Slices, domain.SliceRevisionRecord{TaskID: task, Revision: revision, Segments: []domain.SliceSegment{{Start: 0, End: float64(revision + 10)}}}) + } + } + needed := SliceVersions(d) + if len(needed) != 2 || !needed["task"][2] || !needed["task"][3] || !needed["legacy"][7] { + t.Fatalf("incorrect needed versions: %v", needed) + } + before := Build(d, Query{}) + pruned := d + pruned.Slices = nil + for _, s := range d.Slices { + if needed[s.TaskID][s.Revision] { + pruned.Slices = append(pruned.Slices, s) + } + } + if len(pruned.Slices) != 3 { + t.Fatalf("read %d versions instead of 3", len(pruned.Slices)) + } + if after := Build(pruned, Query{}); !reflect.DeepEqual(before, after) { + t.Fatal("report changed after excluding unused historical slice versions") + } + d.AcceptanceDecisions[0].ReviewVideoSources = []domain.ReviewVideoSource{{SourceItemID: "source", TaskID: "legacy", VideoSeconds: &frozen}} + if SliceVersions(d)["legacy"][7] { + t.Fatal("frozen acceptance still requests historical slices") + } +}