diff --git a/.scratch/query-readmodels/work.md b/.scratch/query-readmodels/work.md index e6b68e1a..0d44d467 100644 --- a/.scratch/query-readmodels/work.md +++ b/.scratch/query-readmodels/work.md @@ -6,3 +6,20 @@ Base: origin/dev @ 5375074a, fast-forwarded to deployed 4170c74b before changes Scope: Query summaries/search planning, explicit list summaries and scoped counts, reliable derived-data maintenance/backfill, incremental reporting/snapshot reads, regression/performance evidence. Out of scope: Production data/index writes, push, PR and deployment until separately authorized; unrelated local work; permission relaxation, truncated history or changed search semantics. Risk: Derived-state freshness and recovery, mixed clients, old payload search, authoritative claim permission, ordering/cursors, dates and supplier/source deduplication. Query speed must meet representative isolated concurrent p95 <= 2 seconds before release preparation; functional checks alone are insufficient. + +## Implementation and compatibility + +- Source-owned `poolSearch` fields are stamped atomically with HumanItem/Task/Project/Run/Capture writes after supplier scrubbing. Search narrows candidates before the existing joins/sorts; the final matcher and server eligibility remain authoritative. Missing/stale metadata uses the existing pipeline. Broad reference sets never truncate results. +- Explicit `/api/human-item-summaries` and `/api/human-item-counts` protocols; legacy item API remains intact. Acceptance detail lives separately from list summaries. Counts traverse all authorized pages and do not return partial totals after errors. +- Summary backfill is an explicit, read-only-by-default CLI. Full-document CAS, idempotent reruns, counters only, 64 KiB per-document summary ceiling; no startup/GET writes, index changes or customer-body rewrites. Production apply still needs authorization. Old servers invalidate summaries through source guards; state-only review updates preserve valid summaries without rewriting annotations. +- Statistics reuse the existing shared immutable fact snapshots and add bounded (4 MiB / 128 entries) generation/query-scoped compressed result reuse and concurrent request coalescing. Personal/supplier scope, dates, timezone identity and generation are in the key; decoding/calculation remain bounded. This does NOT implement event-driven source ingestion: existing 120-second projected-source refresh/recovery stays in place. Its 5,000-part/25,000-event cold build measured 240 ms locally; there is no measured reason yet to replace that recovery path. + +## Evidence so far (not release acceptance) + +- Production base 4170c74b, fresh source preserved. Main and unrelated worktrees untouched. +- Mongo modern/legacy search parity, payload-only legacy names, tenant poisoning, literal regex characters, category/cursor cases, dry run/retry, old-writer invalidation/rebuild, project rename and state-only review metadata passed locally. +- API summary/full metadata equivalence, shared acceptance revocation and >1,000-row counts passed. Browser search/pagination/retry/unvisited-tab totals passed; quality/acceptance/legacy receipt focus and unsaved-opinion flows passed. +- Dense isolated comparison, identical 81 actors, source payloads, 10-second warmup/60-second measurement: base had 52 errors; candidate 1,765 requests/zero errors. Search p95 7.4–12.4 s -> 257–358 ms; common successful identity sets matched for 1,691 samples, with no differences. Some base actor/endpoint pairs had no successful reference and remain covered by separate equivalence tests, not by that comparison. +- Mongo reply bytes 7.21 GB -> 0.119 GB in that synthetic mix; do not extrapolate it into production savings. Compression settings fixed; no media. CPU/heap readings include the in-process harness, not standalone Pod capacity. +- Statistics isolated 5,000 parts/25,000 events: cold build 240 ms, warm p95 30 ms, 121-request/4-handler cohort burst p95 60 ms, zero history reloads during warm burst. +- First task gate caught legacy no-hit foreign-ID transfer exceeding 64 KiB. Fixed with a tiny legacy-readiness probe; legacy/modern/large-payload Mongo and snapshot regressions passed afterwards. Final task/integration and prolonged repeated performance checks remain pending. Keep this branch local; no production changes, push, PR or deployment authorized for this task. diff --git a/cmd/pool-search-backfill/main.go b/cmd/pool-search-backfill/main.go new file mode 100644 index 00000000..e6f20bf1 --- /dev/null +++ b/cmd/pool-search-backfill/main.go @@ -0,0 +1,46 @@ +// Explicit derived-field maintenance. Production execution requires the +// project's separate customer-data write authorization; default is read-only. +package main + +import ( + "context" + "droboticflow/internal/store" + "encoding/json" + "flag" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "time" +) + +func main() { + database := flag.String("database", "", "database name") + org := flag.String("organization", "", "one organization ID") + uriEnv := flag.String("uri-env", "POOL_BACKFILL_MONGO_URI", "environment variable containing Mongo URI") + apply := flag.Bool("apply", false, "write derived search fields; requires prior authorization for production") + timeout := flag.Duration("timeout", 30*time.Minute, "total operation deadline") + flag.Parse() + if *database == "" || *org == "" || os.Getenv(*uriEnv) == "" || *timeout <= 0 { + fmt.Fprintln(os.Stderr, "database, organization and URI environment variable are required") + os.Exit(2) + } + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv(*uriEnv))) + if err != nil { + fmt.Fprintln(os.Stderr, "Mongo initialization failed") + os.Exit(1) + } + defer client.Disconnect(context.Background()) + result, err := store.NewMongoStore(client.Database(*database)).BackfillPoolSearchMetadata(ctx, *org, *apply) + // Diagnostics contain counters only, never records or credential-bearing errors. + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"apply": *apply, "counts": result, "complete": err == nil && result.Conflicts == 0 && result.Unsupported == 0 && (*apply || result.Scanned == 0)}) + if err != nil { + fmt.Fprintln(os.Stderr, "backfill failed; rerun after resolving the connection or timeout") + os.Exit(1) + } + if result.Conflicts > 0 || result.Unsupported > 0 { + os.Exit(2) + } +} diff --git a/internal/api/annotation_output.go b/internal/api/annotation_output.go index 9d4d8c6a..ac5150ca 100644 --- a/internal/api/annotation_output.go +++ b/internal/api/annotation_output.go @@ -62,7 +62,7 @@ func outputDateRange(r *http.Request, now time.Time) (time.Time, time.Time, *tim if zone == "" { zone = "Asia/Shanghai" } - loc, e := time.LoadLocation(zone) + loc, e := workforcestats.ReportingLocation(zone) if e != nil { return time.Time{}, time.Time{}, nil, fmt.Errorf("invalid time zone") } diff --git a/internal/api/human_display.go b/internal/api/human_display.go index 1e010716..03f9c190 100644 --- a/internal/api/human_display.go +++ b/internal/api/human_display.go @@ -224,7 +224,11 @@ func (resolver *humanDisplayResolver) enrichItem(ctx context.Context, item *doma } if len(values) == 0 { appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(item.Subject.ItemKey)) - humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values) + if item.PoolSearchCurrent() { + values = append(values, item.PoolSearch.PayloadDirectories...) + } else { + humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values) + } } // Task artifacts are a legacy fallback, not another source to merge with // an already resolved capture. Avoid fetching the full annotation document diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 272c9cea..8185c089 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -491,7 +491,7 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht if nextCursor != nil && !exhaustedAll && !hasMore && len(filtered) < limit { hasMore = true } - if pool == "reviewed" { + if pool == "reviewed" && poolReadMode(request) != humanPoolReadCount { ids := make([]string, 0, len(filtered)) for _, item := range filtered { ids = append(ids, item.ID) @@ -517,6 +517,13 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht } } } + if poolReadMode(request) == humanPoolReadCount { + page := &humanItemListPage{Items: filtered, HasMore: hasMore} + if hasMore { + page.NextCursor = encodeHumanItemCursor(nextCursor) + } + return page, request + } resolver := newHumanDisplayResolver(server) if err := resolver.primeRuns(request.Context(), filtered); err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -528,7 +535,12 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht return nil, request } server.anonymizeV58AcceptanceItem(request.Context(), &filtered[index]) - if err := server.projectRejectedAcceptanceData(request.Context(), &filtered[index]); err != nil { + if err := func() error { + if poolReadMode(request) == humanPoolReadSummary { + return nil + } + return server.projectRejectedAcceptanceData(request.Context(), &filtered[index]) + }(); err != nil { renderError(writer, http.StatusInternalServerError, err) return nil, request } @@ -618,6 +630,9 @@ func (server *server) filterHumanPoolItem(request *http.Request, item domain.Hum // changes between reads sends this page through the original single-read path; // stale metadata must not govern a newer assignment, stage or permission scope. func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain.HumanItemRecord, pool string, query domain.HumanItemQuery, remaining int, batches map[string]*domain.HumanBatchRecord) ([]domain.HumanItemRecord, error) { + if poolReadMode(request) == humanPoolReadCount { + return items, nil + } ids := []string{} selected := map[string]int{} for index, item := range items { @@ -626,6 +641,9 @@ func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain return nil, err } if visible { + if poolReadMode(request) == humanPoolReadSummary && item.PoolSearchCurrent() { + continue + } ids = append(ids, item.ID) selected[item.ID] = index if len(ids) == remaining { diff --git a/internal/api/human_pool_reads.go b/internal/api/human_pool_reads.go new file mode 100644 index 00000000..9e147bbe --- /dev/null +++ b/internal/api/human_pool_reads.go @@ -0,0 +1,207 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "net/http" +) + +type humanPoolReadMode int + +const ( + humanPoolReadFull humanPoolReadMode = iota + humanPoolReadSummary + humanPoolReadCount +) + +type humanPoolReadModeKey struct{} + +func poolReadMode(r *http.Request) humanPoolReadMode { + mode, _ := r.Context().Value(humanPoolReadModeKey{}).(humanPoolReadMode) + return mode +} + +// Summary is an explicit opt-in protocol. Legacy clients retain their item +// response; workbenches must fetch detail before opening a summary. +type humanPoolSummary struct { + Kind string `json:"kind"` + DurationSeconds float64 `json:"durationSeconds,omitempty"` + SourceValidationStatus string `json:"sourceValidationStatus,omitempty"` + TraceSummary *domain.HumanTraceSummary `json:"traceSummary,omitempty"` + ReviewHistory *domain.HumanReviewHistory `json:"reviewHistory,omitempty"` + CanClaim *bool `json:"canClaim,omitempty"` + AcceptanceClaim *domain.AcceptanceClaimAccess `json:"acceptanceClaim,omitempty"` + ReviewHistoryStatus string `json:"reviewHistoryStatus,omitempty"` + ReviewOperationID string `json:"reviewOperationId,omitempty"` + ProcessingScope string `json:"processingScope,omitempty"` + ProcessingProfile string `json:"processingProfile,omitempty"` + ID string `json:"id"` + OrganizationID string `json:"organizationId"` + PluginID string `json:"pluginId"` + BatchID string `json:"batchId"` + ProjectID string `json:"projectId"` + SupplierID string `json:"supplierId,omitempty"` + WorkflowID string `json:"workflowId"` + RunID string `json:"runId"` + GateNodeID string `json:"gateNodeId"` + LegacyTaskID string `json:"legacyTaskId,omitempty"` + LegacyAuditTaskID string `json:"legacyAuditTaskId,omitempty"` + Subject domain.HumanSubjectRef `json:"subject"` + CurrentStageID string `json:"currentStageId"` + Status domain.HumanItemStatus `json:"status"` + AssignedUserID string `json:"assignedUserId,omitempty"` + LegacyAssignment map[string]string `json:"legacyAssignment,omitempty"` + CurrentRevision int `json:"currentRevision"` + ReviewRound int `json:"reviewRound"` + MaxReworkRounds int `json:"maxReworkRounds"` + Version int `json:"version"` + LatestAuthorUserID string `json:"latestAuthorUserId,omitempty"` + PendingCombinedSubmissionID string `json:"pendingCombinedSubmissionId,omitempty"` + PendingCombinedPayloadSHA string `json:"pendingCombinedPayloadSha,omitempty"` + ReleasePartitionID string `json:"releasePartitionId,omitempty"` + ClaimedAt string `json:"claimedAt,omitempty"` + SubmittedAt string `json:"submittedAt,omitempty"` + AcceptanceSampleProbability *float64 `json:"acceptanceSampleProbability,omitempty"` + AcceptanceSampleValue *float64 `json:"acceptanceSampleValue,omitempty"` + AcceptancePolicyRevision int `json:"acceptancePolicyRevision,omitempty"` + AcceptedAt string `json:"acceptedAt,omitempty"` + AcceptedByUserID string `json:"acceptedByUserId,omitempty"` + RejectedAt string `json:"rejectedAt,omitempty"` + CancelledAt string `json:"cancelledAt,omitempty"` + CancelledByUserID string `json:"cancelledByUserId,omitempty"` + CancellationReason string `json:"cancellationReason,omitempty"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Display *domain.HumanTaskDisplayContext `json:"display,omitempty"` +} + +func summarizeHumanPoolItem(i domain.HumanItemRecord) humanPoolSummary { + seconds := float64(0) + if i.PoolSearchCurrent() { + seconds = i.PoolSearch.DurationSeconds + } else { + for _, value := range listValue(i.Subject.Payload["items"]) { + seconds += max(0, floatValue(mapValue(mapValue(mapValue(value)["materialization"])["coverage"])["durationSeconds"])) + } + } + subject := i.Subject + subject.Payload = nil + return humanPoolSummary{Kind: "pool-summary", DurationSeconds: seconds, + SourceValidationStatus: i.SourceValidationStatus, + TraceSummary: i.TraceSummary, + ReviewHistory: i.ReviewHistory, + CanClaim: i.CanClaim, + AcceptanceClaim: i.AcceptanceClaim, + ReviewHistoryStatus: i.ReviewHistoryStatus, + ReviewOperationID: i.ReviewOperationID, + ProcessingScope: i.ProcessingScope, + ProcessingProfile: i.ProcessingProfile, + ID: i.ID, + OrganizationID: i.OrganizationID, + PluginID: i.PluginID, + BatchID: i.BatchID, + ProjectID: i.ProjectID, + SupplierID: i.SupplierID, + WorkflowID: i.WorkflowID, + RunID: i.RunID, + GateNodeID: i.GateNodeID, + LegacyTaskID: i.LegacyTaskID, + LegacyAuditTaskID: i.LegacyAuditTaskID, + Subject: subject, + CurrentStageID: i.CurrentStageID, + Status: i.Status, + AssignedUserID: i.AssignedUserID, + LegacyAssignment: i.LegacyAssignment, + CurrentRevision: i.CurrentRevision, + ReviewRound: i.ReviewRound, + MaxReworkRounds: i.MaxReworkRounds, + Version: i.Version, + LatestAuthorUserID: i.LatestAuthorUserID, + PendingCombinedSubmissionID: i.PendingCombinedSubmissionID, + PendingCombinedPayloadSHA: i.PendingCombinedPayloadSHA, + ReleasePartitionID: i.ReleasePartitionID, + ClaimedAt: i.ClaimedAt, + SubmittedAt: i.SubmittedAt, + AcceptanceSampleProbability: i.AcceptanceSampleProbability, + AcceptanceSampleValue: i.AcceptanceSampleValue, + AcceptancePolicyRevision: i.AcceptancePolicyRevision, + AcceptedAt: i.AcceptedAt, + AcceptedByUserID: i.AcceptedByUserID, + RejectedAt: i.RejectedAt, + CancelledAt: i.CancelledAt, + CancelledByUserID: i.CancelledByUserID, + CancellationReason: i.CancellationReason, + CreatedAt: i.CreatedAt, + UpdatedAt: i.UpdatedAt, + Display: i.Display, + } +} +func (s *server) handleHumanPoolSummaries(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(context.WithValue(r.Context(), humanPoolReadModeKey{}, humanPoolReadSummary)) + page, _ := s.readHumanItemsList(w, r) + if page == nil { + return + } + summaries := make([]humanPoolSummary, 0, len(page.Items)) + for _, item := range page.Items { + summaries = append(summaries, summarizeHumanPoolItem(item)) + } + renderJSON(w, 200, map[string]any{"summaries": summaries, "filtersApplied": true, "hasMore": page.HasMore, "nextCursor": page.NextCursor}) +} + +// Count every authorized page, without payload hydration or display joins. +// Errors/cancellation return an error, never a partial total. No cross-request +// cache: revocation and self-review rules are checked by the same list filter. +func (s *server) handleHumanPoolCounts(w http.ResponseWriter, r *http.Request) { + counts := map[string]int{} + pools := []string{"available", "mine"} + principal, _ := currentPrincipal(r) + if annotationManager(principal) || principal != nil && (principal.User.Role == domain.UserRoleQualityReviewer || principal.User.Role == domain.UserRoleAcceptanceReviewer) { + pools = append(pools, "reviewed") + } + if annotationManager(principal) { + pools = append(pools, "all") + } + for _, pool := range pools { + count := 0 + cursor := "" + for { + if r.Context().Err() != nil { + renderError(w, 500, r.Context().Err()) + return + } + next := r.Clone(context.WithValue(r.Context(), humanPoolReadModeKey{}, humanPoolReadCount)) + u := *r.URL + next.URL = &u + query := next.URL.Query() + if pool == "reviewed" { + query.Del("stageId") + query.Del("stageIds") + if stages := query.Get("reviewedStageIds"); stages != "" { + query.Set("stageIds", stages) + } + } + query.Del("reviewedStageIds") + query.Set("pool", pool) + query.Set("limit", "1000") + query.Set("cursor", cursor) + next.URL.RawQuery = query.Encode() + page, readRequest := s.readHumanItemsList(w, next) + if page == nil { + return + } + r = r.WithContext(readRequest.Context()) + count += len(page.Items) + if !page.HasMore { + break + } + if page.NextCursor == "" || page.NextCursor == cursor { + renderJSON(w, 500, map[string]string{"error": "任务数量暂时无法核对,请重试。"}) + return + } + cursor = page.NextCursor + } + counts[pool] = count + } + renderJSON(w, 200, map[string]any{"counts": counts, "filtersApplied": true}) +} diff --git a/internal/api/human_pool_reads_test.go b/internal/api/human_pool_reads_test.go new file mode 100644 index 00000000..9bbdffb9 --- /dev/null +++ b/internal/api/human_pool_reads_test.go @@ -0,0 +1,101 @@ +package api + +import ( + "droboticflow/internal/domain" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" +) + +func TestHumanPoolSummaryAndCountPreserveAuthorization(t *testing.T) { + f := sharedAcceptanceFixture(t) + f.item.Subject.Payload["hugeDetail"] = strings.Repeat("private draft ", 100000) + if err := f.store.UpsertHumanItem(f.ctx, f.item); err != nil { + t.Fatal(err) + } + full := f.request("GET", "/api/human-items?pluginId=annotation-qa&pool=available", f.reviewer, nil) + summary := f.request("GET", "/api/human-item-summaries?pluginId=annotation-qa&pool=available", f.reviewer, nil) + if full.Code != 200 || summary.Code != 200 { + t.Fatalf("full=%d summary=%d %s", full.Code, summary.Code, summary.Body.String()) + } + var before struct { + Items []domain.HumanItemRecord `json:"items"` + } + var after struct { + Summaries []humanPoolSummary `json:"summaries"` + } + if err := json.Unmarshal(full.Body.Bytes(), &before); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(summary.Body.Bytes(), &after); err != nil { + t.Fatal(err) + } + // Compare the public metadata contract independently of the DTO converter. + var fullJSON, summaryJSON map[string]any + _ = json.Unmarshal(full.Body.Bytes(), &fullJSON) + _ = json.Unmarshal(summary.Body.Bytes(), &summaryJSON) + for index, value := range fullJSON["items"].([]any) { + expected := value.(map[string]any) + delete(expected["subject"].(map[string]any), "payload") + actual := summaryJSON["summaries"].([]any)[index].(map[string]any) + delete(actual, "kind") + delete(actual, "durationSeconds") + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("public list metadata changed: expected=%v actual=%v", expected, actual) + } + } + if len(before.Items) != len(after.Summaries) || len(after.Summaries) == 0 { + t.Fatal("summary lost rows") + } + for i, item := range before.Items { + want := summarizeHumanPoolItem(item) + if !reflect.DeepEqual(want, after.Summaries[i]) { + t.Fatalf("summary metadata changed: %#v %#v", want, after.Summaries[i]) + } + } + if len(summary.Body.Bytes()) >= 10000 || strings.Contains(summary.Body.String(), "private draft") { + t.Fatal("summary returned detail") + } + counts := f.request("GET", "/api/human-item-counts?pluginId=annotation-qa&stageIds=acceptance,prepackage_acceptance&reviewedStageIds=acceptance,prepackage_acceptance", f.reviewer, nil) + var totals struct { + Counts map[string]int `json:"counts"` + } + if counts.Code != 200 || json.Unmarshal(counts.Body.Bytes(), &totals) != nil || totals.Counts["available"] != len(before.Items) { + t.Fatalf("counts=%d %s", counts.Code, counts.Body.String()) + } + if _, err := f.store.SetSupplierAcceptanceAuthorization(f.ctx, f.item.OrganizationID, f.reviewer.SupplierID, false, 1, f.admin.ID); err != nil { + t.Fatal(err) + } + for _, path := range []string{"/api/human-item-summaries?pool=available&pluginId=annotation-qa", "/api/human-item-counts?pluginId=annotation-qa"} { + result := f.request("GET", path, f.reviewer, nil) + if result.Code != 200 || strings.Contains(result.Body.String(), `"canClaim":true`) { + t.Fatalf("revoked result %d %s", result.Code, result.Body.String()) + } + } +} + +func TestHumanPoolCountsTraversesPagesWithoutHydratingPayloads(t *testing.T) { + f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1) + for n := 0; n < 1005; n++ { + item := f.item + item.ID = fmt.Sprintf("count-%04d", n) + item.CreatedAt = time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339Nano) + item.Subject.Payload = map[string]any{"hugeDetail": strings.Repeat("x", 10000)} + if err := f.store.UpsertHumanItem(f.ctx, item); err != nil { + t.Fatal(err) + } + } + result := f.request("GET", "/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review", f.quality, nil) + var body struct { + Counts map[string]int `json:"counts"` + } + if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &body) != nil || body.Counts["available"] != 1006 { + t.Fatalf("count=%d %s", result.Code, result.Body.String()) + } + if result.Body.Len() > 500 { + t.Fatal("count hydrated details") + } +} diff --git a/internal/api/poll_pool_load_test.go b/internal/api/poll_pool_load_test.go index 3f914929..94f94063 100644 --- a/internal/api/poll_pool_load_test.go +++ b/internal/api/poll_pool_load_test.go @@ -84,7 +84,12 @@ func TestPollPoolReadMixLocal(t *testing.T) { if totalActors > 250 { t.Fatal("local read mix supports at most 250 actors") } + availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8) + if availablePerStage > 250 { + t.Fatal("available fixtures limited to 250 per stage and supplier") + } claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1" + readModels := os.Getenv("POLL_POOL_READMODELS") == "1" mode := os.Getenv("POLL_POOL_MODE") if mode == "" { @@ -214,6 +219,14 @@ func TestPollPoolReadMixLocal(t *testing.T) { t.Fatal("run payload must be 0..2048 KiB") } } + taskPayloadKiB := 0 + if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" { + taskPayloadKiB, err = strconv.Atoi(raw) + must(err) + if taskPayloadKiB < 0 || taskPayloadKiB > 2048 { + t.Fatal("task payload must be 0..2048 KiB") + } + } seededRuns := map[string]bool{} seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) { batchID := "batch-" + p + s @@ -237,7 +250,12 @@ func TestPollPoolReadMixLocal(t *testing.T) { } taskID := "task-" + id must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"})) - must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}}})) + itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}} + if taskPayloadKiB > 0 { + content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024] + itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}} + } + must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}})) } for _, a := range actors { if a.Role == "admin" { @@ -269,7 +287,7 @@ func TestPollPoolReadMixLocal(t *testing.T) { stages = append(stages, "annotation_combined") } for _, stage := range stages { - for n := 0; n < 8; n++ { + for n := 0; n < availablePerStage; n++ { seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady) } } @@ -335,6 +353,13 @@ func TestPollPoolReadMixLocal(t *testing.T) { if authorizationState == "initialized" { must(ds.InitializeSupplierProjectAuthorizations(ctx, org)) } + if readModels { + result, err := ds.BackfillPoolSearchMetadata(ctx, org, true) + must(err) + if result.Unsupported > 0 || result.Conflicts > 0 { + t.Fatalf("summary preparation incomplete: %+v", result) + } + } handler := NewHandler(ds, nil) var mu sync.Mutex samples := []pollLoadSample{} @@ -438,7 +463,11 @@ func TestPollPoolReadMixLocal(t *testing.T) { } result := map[string]string{} for _, pool := range []string{"available", "mine", "reviewed"} { - base := "/api/human-items?pluginId=annotation-qa&pool=" + pool + scope + endpoint := "/api/human-items" + if readModels { + endpoint = "/api/human-item-summaries" + } + base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope if a.Role == "quality" { result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review" } else { @@ -447,6 +476,26 @@ func TestPollPoolReadMixLocal(t *testing.T) { } } } + if readModels && countCycle { + stages := "quality_review,semantic_review" + reviewed := "" + if a.Role == "acceptance" { + stages = "acceptance,prepackage_acceptance" + reviewed = "&reviewedStageIds=" + stages + } + result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed + } + if os.Getenv("POLL_POOL_SEARCH") == "1" { + endpoint := "/api/human-items" + if readModels { + endpoint = "/api/human-item-summaries" + } + stage := "quality_review" + if a.Role == "acceptance" { + stage = "prepackage_acceptance" + } + result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0" + } return result } // Fail before timing if permissions or fixtures accidentally produce empty work. @@ -610,7 +659,7 @@ func TestPollPoolReadMixLocal(t *testing.T) { sort.Float64s(lat) metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows} } - report := map[string]any{"schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples} + report := map[string]any{"read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples} encoded, e := json.MarshalIndent(report, "", " ") must(e) must(os.WriteFile(out, encoded, 0600)) diff --git a/internal/api/production_quality.go b/internal/api/production_quality.go index 792ad770..0194b777 100644 --- a/internal/api/production_quality.go +++ b/internal/api/production_quality.go @@ -21,7 +21,7 @@ func productionQualityDates(r *http.Request, now time.Time) (time.Time, time.Tim if zone == "" { zone = "Asia/Shanghai" } - loc, e := time.LoadLocation(zone) + loc, e := workforcestats.ReportingLocation(zone) if e != nil { return time.Time{}, time.Time{}, nil, e } diff --git a/internal/api/routes_human_loop.go b/internal/api/routes_human_loop.go index eea174d2..1ff1410c 100644 --- a/internal/api/routes_human_loop.go +++ b/internal/api/routes_human_loop.go @@ -8,6 +8,8 @@ import "github.com/go-chi/chi/v5" func (server *server) mountHumanLoopRoutes(router chi.Router) { router.Get("/api/human-batches", server.handleHumanBatchesList) router.Get("/api/human-items", server.handleHumanItemsList) + router.Get("/api/human-item-summaries", server.handleHumanPoolSummaries) + router.Get("/api/human-item-counts", server.handleHumanPoolCounts) router.Get("/api/plugins/annotation-qa/claim-projects", server.handleAnnotationClaimProjects) router.Get("/api/plugins/annotation-qa/claim-count", server.handleAnnotationClaimCount) router.Get("/api/human-items/{id}", server.handleHumanItemGet) diff --git a/internal/api/testdata/routes.golden b/internal/api/testdata/routes.golden index 55869660..bc323526 100644 --- a/internal/api/testdata/routes.golden +++ b/internal/api/testdata/routes.golden @@ -1,7 +1,7 @@ # Generated by TestRouteSnapshot; see routes_snapshot_test.go. # access: reach = endpoint reached; otherwise the status the middleware chain returned. global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache] -routes=379 +routes=381 DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 @@ -67,6 +67,8 @@ GET /api/external/object-sync/sessions/{sessionId}/items | mw=[(*server).require GET /api/external/object-sync/sessions/{sessionId}/status | mw=[(*server).requireObjectSyncFeature-fm,(*server).objectSyncCredentialAuth-fm,(*server).objectSyncDistributedRateLimit.1,(*server).requireObjectSyncIdempotency-fm] | (*server).handleExternalObjectSyncSessionStatus-fm | anon=401 token=401 super=401 admin=401 supplier=401 annotator=401 quality=401 acceptance=401 GET /api/health | mw=[] | (*server).handleHealth-fm | anon=reach token=reach super=reach admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/human-batches | mw=[] | (*server).handleHumanBatchesList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +GET /api/human-item-counts | mw=[] | (*server).handleHumanPoolCounts-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +GET /api/human-item-summaries | mw=[] | (*server).handleHumanPoolSummaries-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/human-items | mw=[] | (*server).handleHumanItemsList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/human-items/{id} | mw=[] | (*server).handleHumanItemGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/human-items/{id}/acceptance-misreports | mw=[(*server).requireBusinessRole.1] | (*server).handleAcceptanceMisreports-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=reach acceptance=reach diff --git a/internal/domain/human_loop.go b/internal/domain/human_loop.go index 0fbd7a14..57556011 100644 --- a/internal/domain/human_loop.go +++ b/internal/domain/human_loop.go @@ -148,7 +148,22 @@ type HumanTraceSummary struct { Rejections *int `json:"rejections"` } +// HumanPoolSearchMetadata contains only source-owned names. It is never public +// task content or a permission decision. +type HumanPoolSearchMetadata struct { + Source map[string]any `bson:"source"` + SourceVersion int `bson:"sourceVersion"` + SourceUpdatedAt string `bson:"sourceUpdatedAt"` + SourceItemKey string `bson:"sourceItemKey"` + SourceArtifactID string `bson:"sourceArtifactId"` + DurationSeconds float64 `bson:"durationSeconds"` + Schema int `bson:"schema"` + Text []string `bson:"text"` + PayloadDirectories []string `bson:"payloadDirectories"` +} + type HumanItemRecord struct { + PoolSearch *HumanPoolSearchMetadata `json:"-" bson:"poolSearch,omitempty"` SourceValidationStatus string `json:"sourceValidationStatus,omitempty" bson:"-"` TraceSummary *HumanTraceSummary `json:"traceSummary,omitempty" bson:"-"` ReviewHistory *HumanReviewHistory `json:"reviewHistory,omitempty" bson:"-"` @@ -497,3 +512,10 @@ type HumanRunDisplay struct { TaskID string `json:"taskId" bson:"taskId"` CaptureDirectories []string `json:"captureDirectories" bson:"captureDirectories"` } + +// PoolSearchCurrent also detects an old server writing a record without +// refreshing its derived summary during a rolling upgrade or rollback. +func (item HumanItemRecord) PoolSearchCurrent() bool { + m := item.PoolSearch + return m != nil && m.Schema == 1 && m.SourceVersion == item.Version && m.SourceUpdatedAt == item.UpdatedAt && m.SourceItemKey == item.Subject.ItemKey && m.SourceArtifactID == item.Subject.ArtifactID +} diff --git a/internal/outputsnapshot/reader.go b/internal/outputsnapshot/reader.go index b8f5497f..0ab979c5 100644 --- a/internal/outputsnapshot/reader.go +++ b/internal/outputsnapshot/reader.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "golang.org/x/sync/singleflight" "io" "log" "sort" @@ -18,24 +19,27 @@ import ( ) type Reader struct { - querySlots chan struct{} - backend Backend - source Source - now func() time.Time - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup - mu sync.Mutex - closed bool - attempts map[string]time.Time - cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache. - cacheBytes int - reads chan struct{} + queryMemo map[string][]byte + queryMemoBytes int + queryFlights singleflight.Group + querySlots chan struct{} + backend Backend + source Source + now func() time.Time + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + mu sync.Mutex + closed bool + attempts map[string]time.Time + cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache. + cacheBytes int + reads chan struct{} } func NewReader(b Backend, source Source) *Reader { ctx, cancel := context.WithCancel(context.Background()) - return &Reader{querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)} + return &Reader{queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)} } func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() } func scopeKey(org string, projects []string) string { @@ -48,7 +52,7 @@ func scopeKey(org string, projects []string) string { // Read returns nil while the initial generation is being built. Existing complete // generations remain available during refresh. Pins never silently change data. -func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) { +func (r *Reader) resolve(ctx context.Context, org string, projects []string, pin string) (*Generation, Info, error) { key, now := scopeKey(org, projects), r.now() state, err := r.backend.OutputSnapshotState(ctx) if err != nil { @@ -98,6 +102,22 @@ func (r *Reader) Read(ctx context.Context, org string, projects []string, pin st } return nil, info, nil } + info.ID = gen.ID + info.AsOf = gen.AsOf + info.Stale = now.Sub(gen.AsOf) >= RefreshInterval + return gen, info, nil +} + +func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) { + gen, info, err := r.resolve(ctx, org, projects, pin) + if err != nil || gen == nil { + return nil, info, err + } + return r.readResolved(ctx, gen, info, pin) +} +func (r *Reader) readResolved(ctx context.Context, gen *Generation, info Info, pin string) (*workforcestats.OutputDataset, Info, error) { + now := r.now() + select { case r.reads <- struct{}{}: case <-ctx.Done(): @@ -105,9 +125,17 @@ func (r *Reader) Read(ctx context.Context, org string, projects []string, pin st } defer func() { <-r.reads }() data, err := r.load(ctx, *gen) - if err != nil && pin == "" && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge { - gen = head.Previous - data, err = r.load(ctx, *gen) + if err != nil && pin == "" { + state, stateErr := r.backend.OutputSnapshotState(ctx) + if stateErr == nil { + for _, head := range state.Heads { + if head.Current != nil && head.Current.ID == gen.ID && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge { + gen = head.Previous + data, err = r.load(ctx, *gen) + break + } + } + } } if err != nil { return nil, info, ErrUnavailable @@ -297,18 +325,137 @@ func (r *Reader) load(ctx context.Context, g Generation) (*workforcestats.Output // Query bounds the entire decode + report calculation, not only Mongo reads. func (r *Reader) Query(ctx context.Context, org string, projects []string, pin string, q workforcestats.OutputQuery) (*workforcestats.OutputReport, Info, error) { + gen, info, err := r.resolve(ctx, org, projects, pin) + if err != nil || gen == nil { + return nil, info, err + } + // Scope permissions are supplied afresh by the handler. The complete query, + // generation and location identity participate in the key, including personal + // and supplier isolation. No permissions or mutable datasets are cached. + q.Now = gen.AsOf + keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)}) + key := string(keyData) select { case r.querySlots <- struct{}{}: case <-ctx.Done(): - return nil, Info{}, ctx.Err() + return nil, info, ctx.Err() } - defer func() { <-r.querySlots }() - d, info, err := r.Read(ctx, org, projects, pin) - if err != nil || d == nil { - return nil, info, err + cached := r.memoReport(key) + <-r.querySlots + if cached != nil { + return cached, info, nil } - result := workforcestats.QueryOutputDataset(*d, q) - return &result, info, nil + type answer struct { + report *workforcestats.OutputReport + info Info + } + result := r.queryFlights.DoChan(key, func() (any, error) { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return nil, ErrUnavailable + } + r.wg.Add(1) + r.mu.Unlock() + defer r.wg.Done() + buildCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second) + defer cancel() + select { + case r.querySlots <- struct{}{}: + case <-buildCtx.Done(): + return nil, buildCtx.Err() + } + defer func() { <-r.querySlots }() + if report := r.memoReport(key); report != nil { + return answer{report, info}, nil + } + data, actual, err := r.readResolved(buildCtx, gen, info, pin) + if err != nil || data == nil { + return nil, err + } + report := workforcestats.QueryOutputDataset(*data, q) + // Corrupt-current fallback must not poison the current generation's key. + if actual.ID == gen.ID { + r.rememberReport(key, &report) + } + return answer{&report, actual}, nil + }) + select { + case <-ctx.Done(): + return nil, info, ctx.Err() + case value := <-result: + if value.Err != nil { + return nil, info, value.Err + } + valueAnswer := value.Val.(answer) + select { + case r.querySlots <- struct{}{}: + case <-ctx.Done(): + return nil, info, ctx.Err() + } + defer func() { <-r.querySlots }() + // Every caller receives an independent value; UI/report handlers may paginate + // or modify the report without mutating another user's response. + raw, err := json.Marshal(valueAnswer.report) + if err != nil { + return nil, info, err + } + var report workforcestats.OutputReport + if err = json.Unmarshal(raw, &report); err != nil { + return nil, info, err + } + return &report, valueAnswer.info, nil + } +} + +const queryMemoBudget = 4 << 20 + +func (r *Reader) memoReport(key string) *workforcestats.OutputReport { + r.mu.Lock() + payload := r.queryMemo[key] + r.mu.Unlock() + if payload == nil { + return nil + } + reader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + return nil + } + defer reader.Close() + var report workforcestats.OutputReport + if json.NewDecoder(io.LimitReader(reader, MaxDecodedBytes+1)).Decode(&report) != nil { + return nil + } + return &report +} +func (r *Reader) rememberReport(key string, report *workforcestats.OutputReport) { + raw, err := json.Marshal(report) + if err != nil || len(raw) > MaxDecodedBytes { + return + } + var encoded bytes.Buffer + writer, _ := gzip.NewWriterLevel(&encoded, gzip.BestSpeed) + if _, err = writer.Write(raw); err != nil { + return + } + if writer.Close() != nil || encoded.Len() > queryMemoBudget { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.queryMemo[key] != nil { + return + } + cost := encoded.Len() + len(key) + 128 + if cost > queryMemoBudget { + return + } + if len(r.queryMemo) >= 128 || r.queryMemoBytes+cost > queryMemoBudget { + r.queryMemo = map[string][]byte{} + r.queryMemoBytes = 0 + } + r.queryMemo[key] = encoded.Bytes() + r.queryMemoBytes += cost } func failureCode(err error) string { diff --git a/internal/outputsnapshot/reader_test.go b/internal/outputsnapshot/reader_test.go index c6ee6179..8cc3ab17 100644 --- a/internal/outputsnapshot/reader_test.go +++ b/internal/outputsnapshot/reader_test.go @@ -267,3 +267,34 @@ func TestSnapshotScopeCountCannotGrowWithoutBound(t *testing.T) { t.Fatal("scope limit not enforced") } } + +func TestReportMemoScopeGenerationAndMutationIsolation(t *testing.T) { + now := time.Now().UTC().Truncate(time.Millisecond) + r := NewReader(&MemoryBackend{}, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) { + return testData(now), nil + }) + defer r.Close() + _, generation := ready(t, r, "") + q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"} + first, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q) + if err != nil || first == nil { + t.Fatalf("query: %v", err) + } + first.Totals.Submitted.Count = -100 + q.Now = now.Add(time.Minute) + second, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q) + if err != nil || second.Totals.Submitted.Count != 12 { + t.Fatalf("memo mutated: %+v %v", second, err) + } + q.ScopeUserID = "other" + other, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q) + if err != nil || other.Totals.Submitted.Count != 0 { + t.Fatalf("scope leaked: %+v %v", other, err) + } + r.mu.Lock() + entries, bytes := len(r.queryMemo), r.queryMemoBytes + r.mu.Unlock() + if entries != 2 || bytes > queryMemoBudget { + t.Fatalf("memo entries=%d bytes=%d", entries, bytes) + } +} diff --git a/internal/store/admin_annotation_pool.go b/internal/store/admin_annotation_pool.go index 9d158ce5..f4bfe79a 100644 --- a/internal/store/admin_annotation_pool.go +++ b/internal/store/admin_annotation_pool.go @@ -169,7 +169,11 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool { } if len(values) == 0 { values = append(values, domain.CaptureDirectoryFromArtifactValue(row.Subject.ItemKey)) - domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values) + if row.PoolSearchCurrent() { + values = append(values, row.PoolSearch.PayloadDirectories...) + } else { + domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values) + } } for _, v := range []string{task.ArtifactID, task.SourceVideo} { values = append(values, domain.CaptureDirectoryFromArtifactValue(v)) diff --git a/internal/store/annotation_artifact.go b/internal/store/annotation_artifact.go index a18aba90..098bcbf5 100644 --- a/internal/store/annotation_artifact.go +++ b/internal/store/annotation_artifact.go @@ -50,6 +50,7 @@ func annotationSafeRunUpdate(stored domain.RunRecord, unset []string) (mongo.Pip if e = bson.Unmarshal(raw, &fields); e != nil { return nil, e } + stampPoolSearchMetadata("runs", fields) set := bson.M{} for key, value := range fields { set[key] = bson.M{"$literal": value} diff --git a/internal/store/capture_automation.go b/internal/store/capture_automation.go index 684a772f..018e80d0 100644 --- a/internal/store/capture_automation.go +++ b/internal/store/capture_automation.go @@ -271,6 +271,7 @@ func (store *MongoStore) upsertCaptureSupplierGuarded(ctx context.Context, item if err != nil { return err } + stampPoolSearchMetadata("captureRecords", set) update := bson.M{"$set": set} if unset := captureRecordUnsetFields(item); len(unset) > 0 { for field := range unset { diff --git a/internal/store/human_pool_search.go b/internal/store/human_pool_search.go index 9b203412..8d02d426 100644 --- a/internal/store/human_pool_search.go +++ b/internal/store/human_pool_search.go @@ -85,6 +85,12 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool { func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) { ctx, cancel := context.WithTimeout(ctx, 8*time.Second) defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } + if q.IDs != nil && len(q.IDs) == 0 || q.AvailableProjectIDs != nil && len(q.AvailableProjectIDs) == 0 { + return []domain.HumanItemRecord{}, nil + } filter := humanItemMongoFilter(org, q) if q.BlindAnnotationActorID != "" { visibility, err := s.blindAnnotationMongoVisibility(ctx, org, q) @@ -93,6 +99,13 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma } filter = bson.M{"$and": bson.A{filter, visibility}} } + candidates, err := s.humanPoolSearchCandidates(ctx, org, q, filter) + if err != nil { + return nil, err + } + if candidates != nil { + filter = bson.M{"$and": bson.A{filter, candidates}} + } pipeline := humanPoolSearchPipeline(org, q, filter) for _, text := range []string{q.Search, q.Capture} { if text != "" { diff --git a/internal/store/human_pool_search_join.go b/internal/store/human_pool_search_join.go index 07785b81..b060f9b5 100644 --- a/internal/store/human_pool_search_join.go +++ b/internal/store/human_pool_search_join.go @@ -13,7 +13,7 @@ import ( func humanPoolSearchPipeline(org string, q domain.HumanItemQuery, filter bson.M) mongo.Pipeline { // Only search/category metadata participates in global sorts and joins. // Hydrate matched IDs afterwards; the public record shape stays unchanged. - projection := bson.M{"_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1} + projection := bson.M{"version": 1, "updatedAt": 1, "poolSearch": 1, "_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1} pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: projection}}} pipeline = append(pipeline, // Number candidate batches; restore the stable createdAt/id order below. diff --git a/internal/store/human_pool_search_payload.go b/internal/store/human_pool_search_payload.go index 004bd8a5..409c4b9f 100644 --- a/internal/store/human_pool_search_payload.go +++ b/internal/store/human_pool_search_payload.go @@ -24,7 +24,7 @@ func (s *MongoStore) humanPoolSearchLegacyPayloads(ctx context.Context, org stri name = domain.CaptureDirectoryFromDirectoryValue(row.Captures[0].Prefix) } } - if name == "" { + if name == "" && !row.PoolSearchCurrent() { ids = append(ids, row.ID) } } diff --git a/internal/store/human_pool_search_test.go b/internal/store/human_pool_search_test.go index 59d9a546..959effc0 100644 --- a/internal/store/human_pool_search_test.go +++ b/internal/store/human_pool_search_test.go @@ -25,7 +25,10 @@ func TestHumanPoolSearchMongoLargeModern(t *testing.T) { func TestHumanPoolSearchMongoLargePayload(t *testing.T) { testHumanPoolSearchMongo(t, 420, true, true) } -func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) { +func TestHumanPoolSearchMongoReadModel(t *testing.T) { + testHumanPoolSearchMongo(t, 10000, true, false, true) +} +func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool, migrate ...bool) { uri := os.Getenv("POOL_LOCAL_MONGO_URI") if uri == "" { t.Skip("isolated local Mongo opt-in") @@ -197,6 +200,20 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) { if _, err := s.annotationTasks.InsertOne(ctx, foreignTask); err != nil { t.Fatal(err) } + if len(migrate) > 0 && migrate[0] { + dry, err := s.BackfillPoolSearchMetadata(ctx, org, false) + if err != nil || dry.Scanned == 0 || dry.Updated != 0 { + t.Fatalf("dry run: %+v %v", dry, err) + } + result, err := s.BackfillPoolSearchMetadata(ctx, org, true) + if err != nil || result.Updated != dry.Scanned || result.Conflicts != 0 || result.Unsupported != 0 { + t.Fatalf("backfill: %+v %v", result, err) + } + again, err := s.BackfillPoolSearchMetadata(ctx, org, true) + if err != nil || again.Scanned != 0 { + t.Fatalf("idempotence: %+v %v", again, err) + } + } ids := func(t *testing.T, rows []domain.HumanItemRecord) []string { out := []string{} for _, i := range rows { @@ -281,4 +298,47 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) { } }) } + if len(migrate) > 0 && migrate[0] { + item, err := s.GetHumanItem(ctx, "i-0008") + if err != nil { + t.Fatal(err) + } + item.Version++ + item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + if _, err = s.humanItems.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": bson.M{"subject.payload.importManifest.sourceDirectory": "archive/moved-legacy-capture", "version": item.Version, "updatedAt": item.UpdatedAt}}); err != nil { + t.Fatal(err) + } + got, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: "moved-legacy-capture", Limit: 200}) + if err != nil || len(got) != 1 || got[0].ID != item.ID { + t.Fatalf("old writer compatibility: %v %v", ids(t, got), err) + } + result, err := s.BackfillPoolSearchMetadata(ctx, org, true) + if err != nil || result.Updated != 1 { + t.Fatalf("repair stale summary: %+v %v", result, err) + } + current, err := s.GetHumanItem(ctx, item.ID) + if err != nil || !current.PoolSearchCurrent() { + t.Fatalf("repaired summary not current: %v", err) + } + next := *current + next.Version++ + next.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + next.Status = domain.HumanItemAccepted + changed, err := s.UpdateHumanItemForReview(ctx, next, current.Version) + if err != nil || !changed { + t.Fatalf("review update: %v %v", changed, err) + } + current, err = s.GetHumanItem(ctx, item.ID) + if err != nil || !current.PoolSearchCurrent() || current.Subject.Payload["importManifest"] == nil { + t.Fatalf("review lost summary or annotation: %v", err) + } + if _, err = s.projects.UpdateOne(ctx, bson.M{"id": "p"}, bson.M{"$set": bson.M{"name": "renamed-project"}}); err != nil { + t.Fatal(err) + } + got, err = s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: "renamed-project", Limit: 200, IdentityProjection: true}) + if err != nil || len(got) != 200 { + t.Fatalf("legacy project rename: count=%d error=%v", len(got), err) + } + } + } diff --git a/internal/store/mongo.go b/internal/store/mongo.go index ac7c743f..05400447 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -529,6 +529,7 @@ func (store *MongoStore) UpsertProject(ctx context.Context, item domain.ProjectR return err } persisted := projectPersistenceDocument(item) + stampPoolSearchMetadata("projects", persisted) _, err = store.projects.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": persisted}, options.Update().SetUpsert(allow)) return err } diff --git a/internal/store/pool_search_backfill.go b/internal/store/pool_search_backfill.go new file mode 100644 index 00000000..de3fa5af --- /dev/null +++ b/internal/store/pool_search_backfill.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type PoolSearchBackfillResult struct{ Scanned, Updated, Conflicts, Unsupported int64 } + +// BackfillPoolSearchMetadata is an explicit maintenance operation, never called +// from GETs or startup. A full-document CAS prevents overwriting any concurrent +// writer, including legacy writers without versions/timestamps. Retry conflicts +// by rerunning; successful records are skipped. No indexes or business fields +// are changed. Dry runs are read-only. +func (s *MongoStore) BackfillPoolSearchMetadata(ctx context.Context, org string, apply bool) (PoolSearchBackfillResult, error) { + result := PoolSearchBackfillResult{} + for _, name := range []string{"humanItems", "runs", "annotationTasks", "captureRecords", "projects"} { + collection := s.db.Collection(name) + filter := bson.M{"organizationId": normalizedOrganizationID(org), "$expr": bson.M{"$not": bson.A{poolSearchValidExpression(name)}}} + cursor, err := collection.Find(ctx, filter, options.Find().SetBatchSize(16)) + if err != nil { + return result, err + } + err = func() error { + defer cursor.Close(ctx) + for cursor.Next(ctx) { + original := append(bson.Raw(nil), cursor.Current...) + var d bson.M + if err := bson.Unmarshal(original, &d); err != nil { + return err + } + result.Scanned++ + stampPoolSearchMetadata(name, d) + metadata := d[poolSearchField] + if metadata == nil { + result.Unsupported++ + continue + } + if !apply { + continue + } + update, err := collection.UpdateOne(ctx, bson.M{"_id": d["_id"], "$expr": bson.M{"$eq": bson.A{"$$ROOT", bson.M{"$literal": original}}}}, bson.M{"$set": bson.M{poolSearchField: metadata}}) + if err != nil { + return fmt.Errorf("backfill %s: %w", name, err) + } + if update.MatchedCount == 0 { + result.Conflicts++ + } else { + result.Updated++ + } + } + return cursor.Err() + }() + if err != nil { + return result, err + } + } + return result, nil +} diff --git a/internal/store/pool_search_metadata.go b/internal/store/pool_search_metadata.go new file mode 100644 index 00000000..706a78de --- /dev/null +++ b/internal/store/pool_search_metadata.go @@ -0,0 +1,330 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "fmt" + "math" + "regexp" + "sort" + "strings" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Owned by the source document, updated in the same write after supplier +// scrubbing. This contains names only, never permissions, eligibility decisions or customer text. +const poolSearchSchema = 1 +const poolSearchField = "poolSearch" + +type poolSearchMetadata = domain.HumanPoolSearchMetadata + +func stampPoolSearchMetadata(collection string, d bson.M) { + m := poolSearchMetadata{Source: map[string]any{}, Schema: poolSearchSchema, Text: []string{}, PayloadDirectories: []string{}} + add := func(keys ...string) { + for _, k := range keys { + if v, ok := d[k].(string); ok { + m.Text = append(m.Text, v) + } + } + } + switch collection { + case "humanItems": + add("id", "legacyTaskId") + subject := adminPoolMap(d["subject"]) + for _, k := range []string{"itemKey", "artifactId"} { + if v, ok := subject[k].(string); ok { + m.Text = append(m.Text, v) + } + } + domain.AppendCaptureDirectoriesFromPayload(subject["payload"], &m.PayloadDirectories) + for _, value := range deletionArray(deletionDoc(subject["payload"])["items"]) { + coverage := deletionDoc(deletionDoc(deletionDoc(value)["materialization"])["coverage"]) + var seconds float64 + switch n := coverage["durationSeconds"].(type) { + case float64: + seconds = n + case int: + seconds = float64(n) + case int32: + seconds = float64(n) + case int64: + seconds = float64(n) + } + if seconds > 0 && !math.IsInf(seconds, 0) && !math.IsNaN(seconds) { + m.DurationSeconds += seconds + } + } + m.Text = append(m.Text, m.PayloadDirectories...) + case "annotationTasks": + add("artifactId", "sourceVideo") + case "captureRecords": + add("captureId", "prefix") + case "projects": + add("name") + case "runs": + // Decode only the fields used by the existing directory parser. + raw, err := bson.Marshal(poolRunSearchDocument(d)) + var run domain.RunRecord + if err != nil || bson.Unmarshal(raw, &run) != nil { + d[poolSearchField] = nil + return + } + m.Text = append(m.Text, domain.CaptureDirectoriesFromRun(run)...) + add("projectName") + default: + return + } + normalize := func(values []string) []string { + seen := map[string]bool{} + out := []string{} + for _, v := range values { + v = strings.ToLower(strings.TrimSpace(v)) + if v != "" && !seen[v] { + seen[v] = true + out = append(out, v) + } + } + sort.Strings(out) + return out + } + for _, value := range append([]string(nil), m.Text...) { + m.Text = append(m.Text, domain.CaptureDirectoryFromArtifactValue(value), domain.CaptureDirectoryFromDirectoryValue(value)) + } + m.Text = normalize(m.Text) + for n, path := range poolSearchSourceFields(collection, d) { + m.Source[fmt.Sprint(n)] = poolSearchSourceValue(d, path) + } + if collection == "humanItems" { + m.SourceVersion = int(supplierNumber(d["version"])) + m.SourceUpdatedAt, _ = d["updatedAt"].(string) + subject := adminPoolMap(d["subject"]) + m.SourceItemKey, _ = subject["itemKey"].(string) + m.SourceArtifactID, _ = subject["artifactId"].(string) + } + // Preserve the exact historical values/order for the final matcher. Only + // prefilter text is normalized; the matcher owns display precedence. + encoded, err := bson.Marshal(m) + if err != nil || len(encoded) > 64<<10 { + d[poolSearchField] = nil + return + } + d[poolSearchField] = m +} + +// A conservative reverse lookup narrows candidates BEFORE joins/sorts. Missing +// summaries always enter the old path; the final matcher and live permissions +// remain authoritative. Very broad reference sets use the bounded old pipeline +// rather than truncating a result or building an unbounded Mongo command. +func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, q domain.HumanItemQuery, base bson.M) (bson.M, error) { + text := q.Capture + if text == "" { + text = q.Search + } + words := strings.Fields(text) + if len(words) == 0 { + return nil, nil + } + // Do not fetch thousands of foreign legacy IDs just to discover that the + // original pipeline is required. A tiny readiness probe preserves the legacy + // no-hit wire budget and keeps mixed-version reads on the proven path. + legacy := bson.M{"$and": bson.A{base, bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression("humanItems")}}}}} + err := s.humanItems.FindOne(ctx, legacy, options.FindOne().SetProjection(bson.M{"_id": 1}).SetMaxTime(8*time.Second)).Err() + if err == nil { + return nil, nil + } + if err != mongo.ErrNoDocuments { + return nil, err + } + token := strings.ToLower(words[0]) + if i := strings.LastIndex(token, "/part"); i > 0 { + token = token[:i] + } + pattern := regexp.QuoteMeta(token) + "|%[0-9a-f]{2}" + matchFor := func(collection string) bson.M { + return bson.M{"$or": bson.A{bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression(collection)}}}, bson.M{poolSearchField + ".text": bson.M{"$regex": pattern}}}} + } + match := matchFor("captureRecords") + const maxReferences = 4096 + ids := func(collection string, extra bson.M) ([]string, bool, error) { + f := bson.M{"organizationId": normalizedOrganizationID(org), "$and": bson.A{match}} + if extra != nil { + f["$and"] = append(f["$and"].(bson.A), extra) + } + cur, err := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetLimit(maxReferences+1).SetMaxTime(8*time.Second)) + if err != nil { + return nil, false, err + } + defer cur.Close(ctx) + out := []string{} + for cur.Next(ctx) { + var v struct { + ID string `bson:"id"` + } + if err := cur.Decode(&v); err != nil { + return nil, false, err + } + out = append(out, v.ID) + } + return out, len(out) > maxReferences, cur.Err() + } + captureIDs, broad, err := ids("captureRecords", nil) + if err != nil || broad { + return nil, err + } + // A capture match follows only same-organization runs. Tenant checks in the + // existing join are retained even after this candidate narrowing. + runMatch := matchFor("runs") + if len(captureIDs) > 0 { + runMatch = bson.M{"$or": bson.A{runMatch, bson.M{"taskId": bson.M{"$in": captureIDs}}}} + } + match = runMatch + runIDs, broad, err := ids("runs", nil) + if err != nil || broad { + return nil, err + } + match = matchFor("annotationTasks") + taskIDs, broad, err := ids("annotationTasks", nil) + if err != nil || broad { + return nil, err + } + match = matchFor("projects") + projectIDs, broad, err := ids("projects", nil) + if err != nil || broad { + return nil, err + } + branches := bson.A{matchFor("humanItems")} + for field, values := range map[string][]string{"runId": runIDs, "legacyTaskId": taskIDs, "projectId": projectIDs} { + if len(values) > 0 { + branches = append(branches, bson.M{field: bson.M{"$in": values}}) + } + } + return bson.M{"$or": branches}, nil +} + +func poolSearchSourceValue(d bson.M, path string) any { + if path == "__runReferences" { + return poolRunReferences(d) + } + parts := strings.Split(path, ".") + for _, part := range parts[:len(parts)-1] { + d = deletionDoc(d[part]) + } + return d[parts[len(parts)-1]] +} +func poolSearchSourceFields(collection string, d bson.M) []string { + switch collection { + case "humanItems": + return []string{"version", "updatedAt", "subject.itemKey", "subject.artifactId"} + case "projects": + return []string{"name"} + case "annotationTasks": + return []string{"artifactId", "sourceVideo"} + case "captureRecords": + return []string{"captureId", "prefix"} + case "runs": + return []string{"projectName", "captureDirectories", "__runReferences"} + } + return nil +} +func poolSearchValidExpression(collection string) bson.M { + fields := poolSearchSourceFields(collection, nil) + terms := bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$poolSearch.schema", 0}}, poolSearchSchema}}} + for n, path := range fields { + terms = append(terms, bson.M{"$eq": bson.A{poolSearchSourceExpression(path), bson.M{"$ifNull": bson.A{"$poolSearch.source." + fmt.Sprint(n), nil}}}}) + } + return bson.M{"$and": terms} +} + +func poolSearchSourceExpression(path string) any { + if path != "__runReferences" { + return bson.M{"$ifNull": bson.A{"$" + path, nil}} + } + text := func(value string) bson.M { return bson.M{"$ifNull": bson.A{value, ""}} } + names := func(input string) bson.M { + return bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "as": "artifact", "in": bson.A{text("$$artifact.id"), text("$$artifact.name"), text("$$artifact.path")}}} + } + inputs := bson.M{"$map": bson.M{"input": bson.M{"$sortArray": bson.M{"input": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$inputs", bson.M{}}}}, "sortBy": bson.M{"k": 1}}}, "as": "input", "in": bson.A{"$$input.k", text("$$input.v.ref.captureUnit"), text("$$input.v.ref.prefix"), text("$$input.v.ref.key"), text("$$input.v.ref.manifestKey")}}} + nodes := bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "node", "in": names("$$node.artifacts")}} + return bson.M{"$cond": bson.A{bson.M{"$gt": bson.A{bson.M{"$size": bson.M{"$ifNull": bson.A{"$captureDirectories", bson.A{}}}}, 0}}, bson.A{}, bson.A{inputs, names("$artifacts"), nodes}}} +} +func poolRunReferences(d bson.M) []any { + if len(deletionArray(d["captureDirectories"])) > 0 { + return []any{} + } + text := func(d bson.M, key string) string { v, _ := d[key].(string); return v } + names := func(value any) []any { + out := []any{} + for _, v := range deletionArray(value) { + a := deletionDoc(v) + out = append(out, []any{text(a, "id"), text(a, "name"), text(a, "path")}) + } + return out + } + inputs := deletionDoc(d["inputs"]) + keys := []string{} + for k := range inputs { + keys = append(keys, k) + } + sort.Strings(keys) + refs := []any{} + for _, k := range keys { + ref := deletionDoc(deletionDoc(inputs[k])["ref"]) + refs = append(refs, []any{k, text(ref, "captureUnit"), text(ref, "prefix"), text(ref, "key"), text(ref, "manifestKey")}) + } + nodes := []any{} + for _, v := range deletionArray(d["nodeExecutions"]) { + nodes = append(nodes, names(deletionDoc(v)["artifacts"])) + } + return []any{refs, names(d["artifacts"]), nodes} +} + +func poolRunSearchDocument(d bson.M) bson.M { + names := func(value any) []any { + out := []any{} + for _, value := range deletionArray(value) { + row := deletionDoc(value) + out = append(out, bson.M{"id": row["id"], "name": row["name"], "path": row["path"]}) + } + return out + } + inputs := bson.M{} + for key, value := range deletionDoc(d["inputs"]) { + ref := deletionDoc(deletionDoc(value)["ref"]) + selected := bson.M{} + for _, field := range []string{"captureUnit", "prefix", "key", "manifestKey"} { + if v, ok := ref[field]; ok { + selected[field] = v + } + } + inputs[key] = bson.M{"ref": selected} + } + nodes := []any{} + for _, value := range deletionArray(d["nodeExecutions"]) { + nodes = append(nodes, bson.M{"artifacts": names(deletionDoc(value)["artifacts"])}) + } + return bson.M{"captureDirectories": d["captureDirectories"], "inputs": inputs, "artifacts": names(d["artifacts"]), "nodeExecutions": nodes} +} + +// State-only review writes must keep a valid name summary current without +// re-reading/rewriting customer annotation payloads. Invalid summaries remain +// invalid and take the compatibility path. The version CAS fences this update. +func poolSearchStateUpdate(fields bson.M) mongo.Pipeline { + set := bson.M{} + for key, value := range fields { + set[key] = bson.M{"$literal": value} + } + set[poolSearchField] = bson.M{"$cond": bson.A{ + poolSearchValidExpression("humanItems"), + bson.M{"$mergeObjects": bson.A{"$poolSearch", bson.M{ + "sourceVersion": bson.M{"$literal": fields["version"]}, + "sourceUpdatedAt": bson.M{"$literal": fields["updatedAt"]}, + "source": bson.M{"$mergeObjects": bson.A{"$poolSearch.source", bson.M{"0": bson.M{"$literal": fields["version"]}, "1": bson.M{"$literal": fields["updatedAt"]}}}}, + }}}, + "$poolSearch", + }} + return mongo.Pipeline{bson.D{{Key: "$set", Value: set}}} +} diff --git a/internal/store/pool_search_metadata_test.go b/internal/store/pool_search_metadata_test.go new file mode 100644 index 00000000..5c31f02c --- /dev/null +++ b/internal/store/pool_search_metadata_test.go @@ -0,0 +1,43 @@ +package store + +import ( + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "reflect" + "testing" +) + +func TestPoolSearchMetadataOwnsOnlyNames(t *testing.T) { + doc := supplierDoc(domain.HumanItemRecord{ID: "i", Subject: domain.HumanSubjectRef{ItemKey: "archive/目录%20甲/part0002.mp4", Payload: map[string]any{"private": "secret comment", "nested": map[string]any{"sourceDirectory": "archive/old-capture"}}}}) + stampPoolSearchMetadata("humanItems", doc) + m := doc[poolSearchField].(poolSearchMetadata) + if !reflect.DeepEqual(m.PayloadDirectories, []string{"old-capture"}) { + t.Fatal(m) + } + for _, v := range m.Text { + if v == "secret comment" { + t.Fatal("customer text indexed") + } + } + found := false + for _, v := range m.Text { + if v == "archive/目录%20甲/part0002.mp4" { + found = true + } + } + if !found { + t.Fatal("encoded artifact compatibility", m) + } + row := adminPoolRow{HumanItemRecord: domain.HumanItemRecord{PoolSearch: &m, Subject: domain.HumanSubjectRef{ItemKey: m.SourceItemKey}}} + if !humanPoolSearchRowMatches(row, domain.HumanItemQuery{Capture: "old-capture"}) { + t.Fatal("summary lost legacy directory") + } + // Authoritative capture names keep their precedence over old payload names. + row.Captures = []domain.CaptureRecord{{CaptureID: "authoritative"}} + if humanPoolSearchRowMatches(row, domain.HumanItemQuery{Capture: "old-capture"}) { + t.Fatal("payload overrode capture") + } + if _, err := bson.Marshal(doc); err != nil { + t.Fatal(err) + } +} diff --git a/internal/store/review_decision.go b/internal/store/review_decision.go index e4853927..cc87802c 100644 --- a/internal/store/review_decision.go +++ b/internal/store/review_decision.go @@ -95,7 +95,7 @@ func humanReviewFields(v domain.HumanItemRecord) bson.M { "reviewOperationId": v.ReviewOperationID} } func (s *MongoStore) UpdateHumanItemForReview(ctx context.Context, next domain.HumanItemRecord, expected int) (bool, error) { - r, err := s.humanItems.UpdateOne(ctx, bson.M{"id": next.ID, "organizationId": next.OrganizationID, "version": expected}, bson.M{"$set": humanReviewFields(next)}) + r, err := s.humanItems.UpdateOne(ctx, bson.M{"id": next.ID, "organizationId": next.OrganizationID, "version": expected}, poolSearchStateUpdate(humanReviewFields(next))) return err == nil && r.MatchedCount == 1, err } func (s *MemoryStore) UpdateHumanItemForReview(_ context.Context, next domain.HumanItemRecord, expected int) (bool, error) { diff --git a/internal/store/supplier_deletion_guard.go b/internal/store/supplier_deletion_guard.go index be89131b..61e06125 100644 --- a/internal/store/supplier_deletion_guard.go +++ b/internal/store/supplier_deletion_guard.go @@ -162,12 +162,18 @@ func scrubSupplierDocument(d bson.M, p domain.SupplierDeletionPlan) bool { delete(d, "vendorId") } if changed { + delete(d, poolSearchField) d["updatedAt"] = nowISO() } return changed } func supplierGuardedMongoWrite(ctx context.Context, col *mongo.Collection, id string, item any, write func(context.Context, bson.M) error) error { d := supplierDoc(item) + writeSource := write + write = func(ctx context.Context, document bson.M) error { + stampPoolSearchMetadata(col.Name(), document) + return writeSource(ctx, document) + } if !supplierGuardRelevant(col.Name()) { return write(ctx, d) } diff --git a/internal/workforcestats/report_location.go b/internal/workforcestats/report_location.go new file mode 100644 index 00000000..05fd08b3 --- /dev/null +++ b/internal/workforcestats/report_location.go @@ -0,0 +1,27 @@ +package workforcestats + +import ( + "sync" + "time" +) + +var reportLocations = struct { + sync.Mutex + values map[string]*time.Location +}{values: map[string]*time.Location{}} + +// ReportingLocation shares immutable IANA rules, with bounded retention. This +// makes location identity stable across equivalent report queries without ever +// treating two arbitrary custom time.Location values as equivalent. +func ReportingLocation(name string) (*time.Location, error) { + reportLocations.Lock() + defer reportLocations.Unlock() + if location := reportLocations.values[name]; location != nil { + return location, nil + } + location, err := time.LoadLocation(name) + if err == nil && len(reportLocations.values) < 32 { + reportLocations.values[name] = location + } + return location, err +} diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 8b8cf859..06b7f330 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -434,6 +434,53 @@ "tests": true } ] + }, + { + "name": "query-readmodels", + "paths": [ + "^internal/store/pool_search.*\\.go$", + "^cmd/pool-search-backfill/", + "^internal/store/(mongo|annotation_artifact|capture_automation|supplier_deletion_guard|review_decision)\\.go$", + "^internal/api/(human_display|production_quality|routes_human_loop)\\.go$", + "^internal/api/testdata/routes\\.golden$", + "^internal/api/poll_pool_load_test\\.go$" + ], + "frontend": [], + "go": { + "./internal/api": ["^TestHumanPoolSummary", "^TestHumanPoolCounts", "^TestHumanDisplay", "^TestRouteSnapshot$"], + "./internal/store": ["^TestPoolSearchMetadata"] + }, + "task": [ + { + "argv": ["npm", "run", "test:go:gate"], + "tests": true + }, + { + "argv": ["go", "test", "./cmd/pool-search-backfill", "-count=1"] + }, + { + "argv": [ + "go", + "test", + "./internal/api", + "-run", + "^TestPollPoolReadMixLocal$", + "-count=1", + "-timeout=3m", + "-v" + ], + "env": { + "POLL_POOL_LOAD": "1", + "POLL_POOL_READMODELS": "1", + "POLL_POOL_SEARCH": "1", + "POLL_POOL_AUTHORIZATION_STATE": "initialized", + "POLL_POOL_WARMUP_SECONDS": "5", + "POLL_POOL_DURATION_SECONDS": "30" + }, + "tests": true + } + ], + "warnings": ["本地 81 人读取冒烟不代表持续容量验收;摘要补建生产 apply 需单独授权,旧记录兼容路径须保留。"] } ] } diff --git a/src/components/AcceptanceTaskCard.tsx b/src/components/AcceptanceTaskCard.tsx index b39f5402..a1d206ae 100644 --- a/src/components/AcceptanceTaskCard.tsx +++ b/src/components/AcceptanceTaskCard.tsx @@ -1,3 +1,4 @@ +import { isHumanPoolSummary } from "../types"; import { ReviewListTrace } from "./ReviewTrace"; import type { HumanItem } from "../types"; import { ReviewHistorySummary } from "./ReviewHistorySummary"; @@ -32,10 +33,13 @@ export function AcceptanceTaskCard({ const completed = ["accepted", "released"].includes(item.status); const reworkReview = !completed && item.status !== "rejected" && Number(item.reviewRound || 0) > 0; const entries = Array.isArray(item.subject?.payload?.items) ? item.subject.payload.items : []; - const seconds = entries.reduce( - (total: number, entry: any) => total + Math.max(0, Number(entry.materialization?.coverage?.durationSeconds) || 0), - 0, - ); + const seconds = isHumanPoolSummary(item) + ? item.durationSeconds || 0 + : entries.reduce( + (total: number, entry: any) => + total + Math.max(0, Number(entry.materialization?.coverage?.durationSeconds) || 0), + 0, + ); const action = acceptanceTaskAction({ status: item.status, mine, claimable }); return (
{ - const page = await human.listItemsPage( + const page = await loadHumanPoolList( + human, { pluginId: "annotation-qa", projectId: scope.projectId, diff --git a/src/lib/review-item-selection.js b/src/lib/review-item-selection.js new file mode 100644 index 00000000..ed583591 --- /dev/null +++ b/src/lib/review-item-selection.js @@ -0,0 +1,11 @@ +// List summaries select an identity, never editable content. Keep legacy +// annotation/package detail fallback when no aggregate pool item is selected. +export function reviewItemForSelection(poolItem, aggregateDetail, annotationItem) { + if (!poolItem) return annotationItem; + if (poolItem.kind !== "pool-summary") return poolItem; + return aggregateDetail?.id === poolItem.id && + aggregateDetail.version >= poolItem.version && + aggregateDetail.kind !== "pool-summary" + ? aggregateDetail + : undefined; +} diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 0dd79436..0e9db22d 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -1,3 +1,5 @@ +import { reviewItemForSelection } from "../lib/review-item-selection.js"; +import { isHumanPoolSummary } from "../types"; import { annotationTimelineFailure } from "../lib/annotation-source-validation.js"; import type { AnnotationTimelineFailure } from "../lib/annotation-source-validation"; // Keep the workbench rules ahead of the review framing rules, matching the @@ -749,6 +751,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: const [templates, setTemplates] = useState([]); const [tasks, setTasks] = useState([]); const [humanPools, setHumanPools] = useState(emptyHumanPools); + const [reviewPoolTotals, setReviewPoolTotals] = useState<{ + scope: string; + counts: Record; + } | null>(null); + const [reviewPoolCountError, setReviewPoolCountError] = useState(""); + const [acceptancePoolDetail, setAcceptancePoolDetail] = useState<{ scope: string; item: HumanItem } | null>(null); + const [acceptancePoolDetailError, setAcceptancePoolDetailError] = useState(""); + const [acceptanceDetailRetry, setAcceptanceDetailRetry] = useState(0); // A URL task id is a selection, not ownership; deep-link previews retain their pool. const defaultTaskPoolView = () => initialAnnotationPool({ @@ -1065,6 +1075,39 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: ...(managerAnnotationPool ? [managerPoolFilters] : []), ...(pagedReviewPool ? [adminSearch, taskStatusFilter, requestedCaptureDirectory] : []), ]); + useEffect(() => { + if (!pagedReviewPool) return; + const controller = new AbortController(); + setReviewPoolCountError(""); + void humanLoopApi + .poolCounts( + { + pluginId: "annotation-qa", + projectId: projectId || undefined, + workflowId: requestedWorkflowId || undefined, + runId: requestedRunId || undefined, + search: adminSearch || undefined, + capture: requestedCaptureDirectory || undefined, + category: taskStatusFilter, + stageIds: acceptanceWorkspace + ? ["acceptance", "prepackage_acceptance"] + : humanItemStageIdsForMode(workspaceMode), + reviewedStageIds: acceptanceWorkspace ? ["acceptance", "prepackage_acceptance"] : undefined, + }, + { signal: controller.signal }, + ) + .then((result) => { + if (!controller.signal.aborted) setReviewPoolTotals({ scope: projectDataScope, counts: result.counts }); + }) + .catch((cause) => { + if (!controller.signal.aborted) { + setReviewPoolTotals(null); + if (cause?.status !== 404 && cause?.status !== 405) + setReviewPoolCountError(getActionErrorMessage(cause, "加载任务数量失败")); + } + }); + return () => controller.abort(); + }, [pagedReviewPool, projectDataScope, humanPools]); const reviewPoolViewRef = useRef(taskPoolView); reviewPoolViewRef.current = taskPoolView; const reviewActionScopeRef = useRef(projectDataScope); @@ -2137,8 +2180,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: if (prepackageAcceptanceItems.some((item) => item.id === requestedAcceptanceId)) setExpandedDeliveryPackageId(requestedAcceptanceId); }, [workspaceMode, requestedAcceptanceId, prepackageAcceptanceItems, loading, saving]); - const reviewSessionItem = - prepackageAcceptanceItems.find((item) => item.id === expandedDeliveryPackageId) || detail?.humanItem; + const acceptanceListSelection = prepackageAcceptanceItems.find((item) => item.id === expandedDeliveryPackageId); + const acceptanceDetailSelection = + acceptancePoolDetail?.scope === projectDataScope && + acceptancePoolDetail.item.id === acceptanceListSelection?.id && + acceptancePoolDetail.item.version >= acceptanceListSelection.version + ? acceptancePoolDetail.item + : null; + useEffect(() => { + if (!acceptanceListSelection || !isHumanPoolSummary(acceptanceListSelection)) return; + const controller = new AbortController(); + setAcceptancePoolDetailError(""); + void humanLoopApi + .getItem(acceptanceListSelection.id, { signal: controller.signal }) + .then(({ item }) => { + if (!controller.signal.aborted) setAcceptancePoolDetail({ scope: projectDataScope, item }); + }) + .catch((cause) => { + if (!controller.signal.aborted) setAcceptancePoolDetailError(getActionErrorMessage(cause, "加载验收内容失败")); + }); + return () => controller.abort(); + }, [projectDataScope, acceptanceListSelection?.id, acceptanceListSelection?.version, acceptanceDetailRetry]); + const reviewSessionItem = reviewItemForSelection( + acceptanceListSelection, + acceptanceDetailSelection, + detail?.humanItem, + ); const reviewSessionEnabled = Boolean( (workspaceMode === "quality" || workspaceMode === "clip-quality" || workspaceMode === "acceptance") && taskPoolView !== "reviewed" && @@ -2817,8 +2884,18 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: : { available: 0, mine: 0, all: 0 }), reviewed: 0, } - : poolTabCounts(humanPools, matchesPoolScope), - [humanPools, matchesPoolScope, managerAnnotationPool, adminPoolTotals, projectDataScope], + : pagedReviewPool && reviewPoolTotals?.scope === projectDataScope + ? reviewPoolTotals.counts + : poolTabCounts(humanPools, matchesPoolScope), + [ + humanPools, + matchesPoolScope, + managerAnnotationPool, + adminPoolTotals, + projectDataScope, + pagedReviewPool, + reviewPoolTotals, + ], ); const stageEmptyLabel = workspaceMode === "quality" @@ -7043,9 +7120,9 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: aria-pressed={taskPoolView === value} className={`rounded-lg border px-3 py-2 text-xs transition-colors ${taskPoolView === value ? "border-blue-500 bg-blue-50 text-blue-600 dark:bg-blue-500/10 dark:text-blue-200 font-medium" : "border-[color:var(--app-border)] text-slate-600 hover:text-slate-900 dark:text-gray-400 dark:hover:text-white"}`} > - {label} {pagedReviewPool ? "已加载 " : ""} + {label} {(managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope) || - (pagedReviewPool && !(value in reviewPoolCursorsRef.current)) + (pagedReviewPool && reviewPoolTotals?.scope !== projectDataScope) ? "—" : blindAnnotator && value === "available" ? claimableCount.projectId === projectId @@ -7053,10 +7130,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: : "—" : count} {/* Counts only loaded pages; a further page means more exist. */} - {(pagedReviewPool && reviewPoolCursorsRef.current[value]) || - (blindAnnotator && value === "mine" && minePoolCursorRef.current) - ? "+" - : ""} + {blindAnnotator && value === "mine" && minePoolCursorRef.current ? "+" : ""} ))} @@ -7183,6 +7257,11 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
逐段任务
+ {reviewPoolCountError && ( +
+ {reviewPoolCountError} +
+ )}
{sidebarLoading && (
@@ -7582,7 +7661,27 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: )} {prepackageAcceptanceItems .filter((i) => i.id === expandedDeliveryPackageId) - .map((item) => { + .map((poolItem) => { + const item = isHumanPoolSummary(poolItem) ? acceptanceDetailSelection : poolItem; + if (!item) + return ( +
+ {acceptancePoolDetailError || "正在加载验收内容…"} + {acceptancePoolDetailError && ( + + )} +
+ ); const stats = prepackageAcceptanceDrafts[item.id] || { ...emptyPrepackageAcceptanceStatistics, headpose: "include", diff --git a/src/services/api/human-loop.ts b/src/services/api/human-loop.ts index 003aa089..c5fd5300 100644 --- a/src/services/api/human-loop.ts +++ b/src/services/api/human-loop.ts @@ -1,4 +1,10 @@ -import type { HumanItem, HumanItemStatus, HumanReject, RunArtifactManifestMembersResponse } from "../../types"; +import type { + HumanItem, + HumanPoolSummary, + HumanItemStatus, + HumanReject, + RunArtifactManifestMembersResponse, +} from "../../types"; import { mergePagedManifestMembers } from "../../lib/annotation-workbench.js"; import { apiRequest } from "./client"; @@ -89,6 +95,19 @@ async function listAcceptancePool(filters: HumanItemFilters, options: { signal?: export const humanLoopApi = { listItems, listItemsPage, + listSummariesPage: (filters: HumanItemFilters = {}, options: { signal?: AbortSignal } = {}) => + apiRequest<{ summaries: HumanPoolSummary[]; filtersApplied: boolean; hasMore: boolean; nextCursor?: string }>( + `/api/human-item-summaries${queryString(filters)}`, + options, + ), + poolCounts: ( + filters: HumanItemFilters & { reviewedStageIds?: string[] } = {}, + options: { signal?: AbortSignal } = {}, + ) => + apiRequest<{ counts: Record<"available" | "mine" | "reviewed" | "all", number>; filtersApplied: boolean }>( + `/api/human-item-counts${queryString(filters)}`, + options, + ), listAllItems, listAcceptancePools: async ( filters: Omit, @@ -110,11 +129,11 @@ export const humanLoopApi = { ); return { available, mine, all: [], reviewed }; }, - getItem: (id: string) => + getItem: (id: string, options: { signal?: AbortSignal } = {}) => apiRequest<{ item: HumanItem; acceptanceReviews?: { taskId: string; revision: number; decision: string; comment?: string; createdAt: string }[]; - }>(`/api/human-items/${encodeURIComponent(id)}`), + }>(`/api/human-items/${encodeURIComponent(id)}`, options), subjectContentURL: (id: string, side: "left" | "right" = "left") => `/api/human-items/${encodeURIComponent(id)}/subject-content${side === "right" ? "?side=right" : ""}`, evidenceContentURL: (id: string, artifactId: string) => diff --git a/src/types.ts b/src/types.ts index cbbf6432..5ccf3c7d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2266,6 +2266,15 @@ export interface HumanItem { display?: HumanTaskDisplayContext; } +/** Explicit list-only identity. A workbench must load HumanItem detail first. */ +export type HumanPoolSummary = Omit & { + kind: "pool-summary"; + durationSeconds?: number; + subject: Omit; +}; +export const isHumanPoolSummary = (item: HumanItem | HumanPoolSummary): item is HumanPoolSummary => + "kind" in item && item.kind === "pool-summary"; + export interface HumanReject { id: string; organizationId: string; diff --git a/tests/browser/review-pool-search-replica.mjs b/tests/browser/review-pool-search-replica.mjs index b694a1c2..40cb11eb 100644 --- a/tests/browser/review-pool-search-replica.mjs +++ b/tests/browser/review-pool-search-replica.mjs @@ -39,7 +39,7 @@ try { return route.fulfill({ status: 409, json: { error: "任务已被其他人领取。" } }); } if (!["GET", "HEAD"].includes(r.method())) report.writes.push(u.pathname); - if (u.pathname === "/api/human-items") { + if (["/api/human-items", "/api/human-item-summaries"].includes(u.pathname)) { report.requests.push(Object.fromEntries(u.searchParams)); if (u.searchParams.get("pool") === "available" && u.searchParams.get("search") === "FYLX_SEARCH_0612") return route.fulfill({ status: 504, json: { error: "合成可领取池超时" } }); @@ -63,8 +63,13 @@ try { cards = page.getByTestId("annotation-task-card"), more = page.getByTestId("task-pool-show-more"); await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 200); - assert.match(await available.innerText(), /已加载 200\+/); - report.cases.push("bounded initial200 and explicitly loaded count"); + await page.waitForFunction(() => + document.querySelector('[data-testid="human-task-pool-available"]').textContent.includes("612"), + ); + assert.match(await available.innerText(), /612/); + assert.match(await page.getByTestId("human-task-pool-mine").innerText(), /1/); + assert.match(await page.getByTestId("human-task-pool-reviewed").innerText(), /1/); + report.cases.push("bounded initial200 with independent totals for unvisited tabs"); await more.click(); await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 400); report.cases.push("available cursor loads next200"); diff --git a/tests/browser/review-trace.mjs b/tests/browser/review-trace.mjs index 9aabb166..98fa94a9 100644 --- a/tests/browser/review-trace.mjs +++ b/tests/browser/review-trace.mjs @@ -90,7 +90,10 @@ try { continuationWrites.push(route.request().postDataJSON()?.action); if (u.pathname.endsWith("/review-session") && route.request().method() === "POST") sessionWrites.push(route.request().postDataJSON()?.action); - if (blockPoolReads && u.pathname === "/api/human-items") { + if ( + blockPoolReads && + ["/api/human-items", "/api/human-item-summaries", "/api/human-item-counts"].includes(u.pathname) + ) { unexpectedPoolReads.push(u.search); return route.fulfill({ status: 503, json: { error: "Synthetic pool outage during layout change" } }); } diff --git a/tests/frontend/annotation-project-loading.test.mjs b/tests/frontend/annotation-project-loading.test.mjs index c6d57306..064ea10a 100644 --- a/tests/frontend/annotation-project-loading.test.mjs +++ b/tests/frontend/annotation-project-loading.test.mjs @@ -348,3 +348,28 @@ test("my-tasks count marks itself partial while another sidebar page exists", as assert.match(tabs, /blindAnnotator && value === "mine" && minePoolCursorRef\.current/); assert.match(tabs, /\? "\+"\s*:\s*""/); }); + +test("summary protocol preserves identity and falls back only on missing routes", async () => { + const { loadHumanPoolList } = await import("../../src/lib/annotation-project-loading.js"); + let legacy = 0; + const item = { kind: "pool-summary", id: "i", version: 7, canClaim: false, subject: { itemKey: "part" } }; + const human = { + listSummariesPage: async () => ({ summaries: [item], filtersApplied: true }), + listItemsPage: async () => { + legacy++; + return { items: [] }; + }, + }; + assert.equal((await loadHumanPoolList(human, {}, {})).items[0], item); + assert.equal(legacy, 0); + human.listSummariesPage = async () => { + throw Object.assign(new Error("missing"), { status: 404 }); + }; + await loadHumanPoolList(human, {}, {}); + assert.equal(legacy, 1); + human.listSummariesPage = async () => { + throw Object.assign(new Error("denied"), { status: 403 }); + }; + await assert.rejects(loadHumanPoolList(human, {}, {}), /denied/); + assert.equal(legacy, 1); +}); diff --git a/tests/frontend/review-item-selection.test.mjs b/tests/frontend/review-item-selection.test.mjs new file mode 100644 index 00000000..8680f0c7 --- /dev/null +++ b/tests/frontend/review-item-selection.test.mjs @@ -0,0 +1,22 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { reviewItemForSelection } from "../../src/lib/review-item-selection.js"; + +test("aggregate summaries wait for matching full detail before editing", () => { + const summary = { kind: "pool-summary", id: "aggregate", version: 3 }; + const unrelated = { id: "annotation", version: 9, subject: { payload: { draft: "preserved" } } }; + assert.equal(reviewItemForSelection(summary, undefined, unrelated), undefined); + assert.equal(reviewItemForSelection(summary, unrelated, unrelated), undefined); + assert.equal(reviewItemForSelection(summary, { id: "aggregate", version: 2 }, unrelated), undefined); + assert.equal(reviewItemForSelection(summary, summary, unrelated), undefined); + const detail = { id: "aggregate", version: 3, subject: { payload: { items: [{ id: "source" }] } } }; + assert.equal(reviewItemForSelection(summary, detail, unrelated), detail); + assert.equal(detail.subject.payload.items[0].id, "source"); +}); + +test("legacy package and annotation selections keep their existing detail path", () => { + const legacy = { id: "legacy-package-task", version: 2 }; + assert.equal(reviewItemForSelection(undefined, undefined, legacy), legacy); + const full = { id: "aggregate", version: 4, subject: { payload: { items: [] } } }; + assert.equal(reviewItemForSelection(full, undefined, legacy), full); +});