⚡ Optimize task pool reads and scoped reporting

This commit is contained in:
Codex 2026-11-09 02:03:30 -08:00
parent 8c2ae878a2
commit 88526258c9
39 changed files with 1516 additions and 63 deletions

View File

@ -6,3 +6,20 @@ Base: origin/dev @ 5375074a, fast-forwarded to deployed 4170c74b before changes
Scope: Query summaries/search planning, explicit list summaries and scoped counts, reliable derived-data maintenance/backfill, incremental reporting/snapshot reads, regression/performance evidence.
Out of scope: Production data/index writes, push, PR and deployment until separately authorized; unrelated local work; permission relaxation, truncated history or changed search semantics.
Risk: Derived-state freshness and recovery, mixed clients, old payload search, authoritative claim permission, ordering/cursors, dates and supplier/source deduplication. Query speed must meet representative isolated concurrent p95 <= 2 seconds before release preparation; functional checks alone are insufficient.
## Implementation and compatibility
- Source-owned `poolSearch` fields are stamped atomically with HumanItem/Task/Project/Run/Capture writes after supplier scrubbing. Search narrows candidates before the existing joins/sorts; the final matcher and server eligibility remain authoritative. Missing/stale metadata uses the existing pipeline. Broad reference sets never truncate results.
- Explicit `/api/human-item-summaries` and `/api/human-item-counts` protocols; legacy item API remains intact. Acceptance detail lives separately from list summaries. Counts traverse all authorized pages and do not return partial totals after errors.
- Summary backfill is an explicit, read-only-by-default CLI. Full-document CAS, idempotent reruns, counters only, 64 KiB per-document summary ceiling; no startup/GET writes, index changes or customer-body rewrites. Production apply still needs authorization. Old servers invalidate summaries through source guards; state-only review updates preserve valid summaries without rewriting annotations.
- Statistics reuse the existing shared immutable fact snapshots and add bounded (4 MiB / 128 entries) generation/query-scoped compressed result reuse and concurrent request coalescing. Personal/supplier scope, dates, timezone identity and generation are in the key; decoding/calculation remain bounded. This does NOT implement event-driven source ingestion: existing 120-second projected-source refresh/recovery stays in place. Its 5,000-part/25,000-event cold build measured 240 ms locally; there is no measured reason yet to replace that recovery path.
## Evidence so far (not release acceptance)
- Production base 4170c74b, fresh source preserved. Main and unrelated worktrees untouched.
- Mongo modern/legacy search parity, payload-only legacy names, tenant poisoning, literal regex characters, category/cursor cases, dry run/retry, old-writer invalidation/rebuild, project rename and state-only review metadata passed locally.
- API summary/full metadata equivalence, shared acceptance revocation and >1,000-row counts passed. Browser search/pagination/retry/unvisited-tab totals passed; quality/acceptance/legacy receipt focus and unsaved-opinion flows passed.
- Dense isolated comparison, identical 81 actors, source payloads, 10-second warmup/60-second measurement: base had 52 errors; candidate 1,765 requests/zero errors. Search p95 7.4–12.4 s -> 257–358 ms; common successful identity sets matched for 1,691 samples, with no differences. Some base actor/endpoint pairs had no successful reference and remain covered by separate equivalence tests, not by that comparison.
- Mongo reply bytes 7.21 GB -> 0.119 GB in that synthetic mix; do not extrapolate it into production savings. Compression settings fixed; no media. CPU/heap readings include the in-process harness, not standalone Pod capacity.
- Statistics isolated 5,000 parts/25,000 events: cold build 240 ms, warm p95 30 ms, 121-request/4-handler cohort burst p95 60 ms, zero history reloads during warm burst.
- First task gate caught legacy no-hit foreign-ID transfer exceeding 64 KiB. Fixed with a tiny legacy-readiness probe; legacy/modern/large-payload Mongo and snapshot regressions passed afterwards. Final task/integration and prolonged repeated performance checks remain pending. Keep this branch local; no production changes, push, PR or deployment authorized for this task.

View File

@ -0,0 +1,46 @@
// Explicit derived-field maintenance. Production execution requires the
// project's separate customer-data write authorization; default is read-only.
package main
import (
"context"
"droboticflow/internal/store"
"encoding/json"
"flag"
"fmt"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"time"
)
func main() {
database := flag.String("database", "", "database name")
org := flag.String("organization", "", "one organization ID")
uriEnv := flag.String("uri-env", "POOL_BACKFILL_MONGO_URI", "environment variable containing Mongo URI")
apply := flag.Bool("apply", false, "write derived search fields; requires prior authorization for production")
timeout := flag.Duration("timeout", 30*time.Minute, "total operation deadline")
flag.Parse()
if *database == "" || *org == "" || os.Getenv(*uriEnv) == "" || *timeout <= 0 {
fmt.Fprintln(os.Stderr, "database, organization and URI environment variable are required")
os.Exit(2)
}
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv(*uriEnv)))
if err != nil {
fmt.Fprintln(os.Stderr, "Mongo initialization failed")
os.Exit(1)
}
defer client.Disconnect(context.Background())
result, err := store.NewMongoStore(client.Database(*database)).BackfillPoolSearchMetadata(ctx, *org, *apply)
// Diagnostics contain counters only, never records or credential-bearing errors.
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"apply": *apply, "counts": result, "complete": err == nil && result.Conflicts == 0 && result.Unsupported == 0 && (*apply || result.Scanned == 0)})
if err != nil {
fmt.Fprintln(os.Stderr, "backfill failed; rerun after resolving the connection or timeout")
os.Exit(1)
}
if result.Conflicts > 0 || result.Unsupported > 0 {
os.Exit(2)
}
}

View File

@ -62,7 +62,7 @@ func outputDateRange(r *http.Request, now time.Time) (time.Time, time.Time, *tim
if zone == "" {
zone = "Asia/Shanghai"
}
loc, e := time.LoadLocation(zone)
loc, e := workforcestats.ReportingLocation(zone)
if e != nil {
return time.Time{}, time.Time{}, nil, fmt.Errorf("invalid time zone")
}

View File

@ -224,7 +224,11 @@ func (resolver *humanDisplayResolver) enrichItem(ctx context.Context, item *doma
}
if len(values) == 0 {
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(item.Subject.ItemKey))
humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values)
if item.PoolSearchCurrent() {
values = append(values, item.PoolSearch.PayloadDirectories...)
} else {
humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values)
}
}
// Task artifacts are a legacy fallback, not another source to merge with
// an already resolved capture. Avoid fetching the full annotation document

View File

@ -491,7 +491,7 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
if nextCursor != nil && !exhaustedAll && !hasMore && len(filtered) < limit {
hasMore = true
}
if pool == "reviewed" {
if pool == "reviewed" && poolReadMode(request) != humanPoolReadCount {
ids := make([]string, 0, len(filtered))
for _, item := range filtered {
ids = append(ids, item.ID)
@ -517,6 +517,13 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
}
}
}
if poolReadMode(request) == humanPoolReadCount {
page := &humanItemListPage{Items: filtered, HasMore: hasMore}
if hasMore {
page.NextCursor = encodeHumanItemCursor(nextCursor)
}
return page, request
}
resolver := newHumanDisplayResolver(server)
if err := resolver.primeRuns(request.Context(), filtered); err != nil {
renderError(writer, http.StatusInternalServerError, err)
@ -528,7 +535,12 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
return nil, request
}
server.anonymizeV58AcceptanceItem(request.Context(), &filtered[index])
if err := server.projectRejectedAcceptanceData(request.Context(), &filtered[index]); err != nil {
if err := func() error {
if poolReadMode(request) == humanPoolReadSummary {
return nil
}
return server.projectRejectedAcceptanceData(request.Context(), &filtered[index])
}(); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return nil, request
}
@ -618,6 +630,9 @@ func (server *server) filterHumanPoolItem(request *http.Request, item domain.Hum
// changes between reads sends this page through the original single-read path;
// stale metadata must not govern a newer assignment, stage or permission scope.
func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain.HumanItemRecord, pool string, query domain.HumanItemQuery, remaining int, batches map[string]*domain.HumanBatchRecord) ([]domain.HumanItemRecord, error) {
if poolReadMode(request) == humanPoolReadCount {
return items, nil
}
ids := []string{}
selected := map[string]int{}
for index, item := range items {
@ -626,6 +641,9 @@ func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain
return nil, err
}
if visible {
if poolReadMode(request) == humanPoolReadSummary && item.PoolSearchCurrent() {
continue
}
ids = append(ids, item.ID)
selected[item.ID] = index
if len(ids) == remaining {

View File

@ -0,0 +1,207 @@
package api
import (
"context"
"droboticflow/internal/domain"
"net/http"
)
type humanPoolReadMode int
const (
humanPoolReadFull humanPoolReadMode = iota
humanPoolReadSummary
humanPoolReadCount
)
type humanPoolReadModeKey struct{}
func poolReadMode(r *http.Request) humanPoolReadMode {
mode, _ := r.Context().Value(humanPoolReadModeKey{}).(humanPoolReadMode)
return mode
}
// Summary is an explicit opt-in protocol. Legacy clients retain their item
// response; workbenches must fetch detail before opening a summary.
type humanPoolSummary struct {
Kind string `json:"kind"`
DurationSeconds float64 `json:"durationSeconds,omitempty"`
SourceValidationStatus string `json:"sourceValidationStatus,omitempty"`
TraceSummary *domain.HumanTraceSummary `json:"traceSummary,omitempty"`
ReviewHistory *domain.HumanReviewHistory `json:"reviewHistory,omitempty"`
CanClaim *bool `json:"canClaim,omitempty"`
AcceptanceClaim *domain.AcceptanceClaimAccess `json:"acceptanceClaim,omitempty"`
ReviewHistoryStatus string `json:"reviewHistoryStatus,omitempty"`
ReviewOperationID string `json:"reviewOperationId,omitempty"`
ProcessingScope string `json:"processingScope,omitempty"`
ProcessingProfile string `json:"processingProfile,omitempty"`
ID string `json:"id"`
OrganizationID string `json:"organizationId"`
PluginID string `json:"pluginId"`
BatchID string `json:"batchId"`
ProjectID string `json:"projectId"`
SupplierID string `json:"supplierId,omitempty"`
WorkflowID string `json:"workflowId"`
RunID string `json:"runId"`
GateNodeID string `json:"gateNodeId"`
LegacyTaskID string `json:"legacyTaskId,omitempty"`
LegacyAuditTaskID string `json:"legacyAuditTaskId,omitempty"`
Subject domain.HumanSubjectRef `json:"subject"`
CurrentStageID string `json:"currentStageId"`
Status domain.HumanItemStatus `json:"status"`
AssignedUserID string `json:"assignedUserId,omitempty"`
LegacyAssignment map[string]string `json:"legacyAssignment,omitempty"`
CurrentRevision int `json:"currentRevision"`
ReviewRound int `json:"reviewRound"`
MaxReworkRounds int `json:"maxReworkRounds"`
Version int `json:"version"`
LatestAuthorUserID string `json:"latestAuthorUserId,omitempty"`
PendingCombinedSubmissionID string `json:"pendingCombinedSubmissionId,omitempty"`
PendingCombinedPayloadSHA string `json:"pendingCombinedPayloadSha,omitempty"`
ReleasePartitionID string `json:"releasePartitionId,omitempty"`
ClaimedAt string `json:"claimedAt,omitempty"`
SubmittedAt string `json:"submittedAt,omitempty"`
AcceptanceSampleProbability *float64 `json:"acceptanceSampleProbability,omitempty"`
AcceptanceSampleValue *float64 `json:"acceptanceSampleValue,omitempty"`
AcceptancePolicyRevision int `json:"acceptancePolicyRevision,omitempty"`
AcceptedAt string `json:"acceptedAt,omitempty"`
AcceptedByUserID string `json:"acceptedByUserId,omitempty"`
RejectedAt string `json:"rejectedAt,omitempty"`
CancelledAt string `json:"cancelledAt,omitempty"`
CancelledByUserID string `json:"cancelledByUserId,omitempty"`
CancellationReason string `json:"cancellationReason,omitempty"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
Display *domain.HumanTaskDisplayContext `json:"display,omitempty"`
}
func summarizeHumanPoolItem(i domain.HumanItemRecord) humanPoolSummary {
seconds := float64(0)
if i.PoolSearchCurrent() {
seconds = i.PoolSearch.DurationSeconds
} else {
for _, value := range listValue(i.Subject.Payload["items"]) {
seconds += max(0, floatValue(mapValue(mapValue(mapValue(value)["materialization"])["coverage"])["durationSeconds"]))
}
}
subject := i.Subject
subject.Payload = nil
return humanPoolSummary{Kind: "pool-summary", DurationSeconds: seconds,
SourceValidationStatus: i.SourceValidationStatus,
TraceSummary: i.TraceSummary,
ReviewHistory: i.ReviewHistory,
CanClaim: i.CanClaim,
AcceptanceClaim: i.AcceptanceClaim,
ReviewHistoryStatus: i.ReviewHistoryStatus,
ReviewOperationID: i.ReviewOperationID,
ProcessingScope: i.ProcessingScope,
ProcessingProfile: i.ProcessingProfile,
ID: i.ID,
OrganizationID: i.OrganizationID,
PluginID: i.PluginID,
BatchID: i.BatchID,
ProjectID: i.ProjectID,
SupplierID: i.SupplierID,
WorkflowID: i.WorkflowID,
RunID: i.RunID,
GateNodeID: i.GateNodeID,
LegacyTaskID: i.LegacyTaskID,
LegacyAuditTaskID: i.LegacyAuditTaskID,
Subject: subject,
CurrentStageID: i.CurrentStageID,
Status: i.Status,
AssignedUserID: i.AssignedUserID,
LegacyAssignment: i.LegacyAssignment,
CurrentRevision: i.CurrentRevision,
ReviewRound: i.ReviewRound,
MaxReworkRounds: i.MaxReworkRounds,
Version: i.Version,
LatestAuthorUserID: i.LatestAuthorUserID,
PendingCombinedSubmissionID: i.PendingCombinedSubmissionID,
PendingCombinedPayloadSHA: i.PendingCombinedPayloadSHA,
ReleasePartitionID: i.ReleasePartitionID,
ClaimedAt: i.ClaimedAt,
SubmittedAt: i.SubmittedAt,
AcceptanceSampleProbability: i.AcceptanceSampleProbability,
AcceptanceSampleValue: i.AcceptanceSampleValue,
AcceptancePolicyRevision: i.AcceptancePolicyRevision,
AcceptedAt: i.AcceptedAt,
AcceptedByUserID: i.AcceptedByUserID,
RejectedAt: i.RejectedAt,
CancelledAt: i.CancelledAt,
CancelledByUserID: i.CancelledByUserID,
CancellationReason: i.CancellationReason,
CreatedAt: i.CreatedAt,
UpdatedAt: i.UpdatedAt,
Display: i.Display,
}
}
func (s *server) handleHumanPoolSummaries(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(context.WithValue(r.Context(), humanPoolReadModeKey{}, humanPoolReadSummary))
page, _ := s.readHumanItemsList(w, r)
if page == nil {
return
}
summaries := make([]humanPoolSummary, 0, len(page.Items))
for _, item := range page.Items {
summaries = append(summaries, summarizeHumanPoolItem(item))
}
renderJSON(w, 200, map[string]any{"summaries": summaries, "filtersApplied": true, "hasMore": page.HasMore, "nextCursor": page.NextCursor})
}
// Count every authorized page, without payload hydration or display joins.
// Errors/cancellation return an error, never a partial total. No cross-request
// cache: revocation and self-review rules are checked by the same list filter.
func (s *server) handleHumanPoolCounts(w http.ResponseWriter, r *http.Request) {
counts := map[string]int{}
pools := []string{"available", "mine"}
principal, _ := currentPrincipal(r)
if annotationManager(principal) || principal != nil && (principal.User.Role == domain.UserRoleQualityReviewer || principal.User.Role == domain.UserRoleAcceptanceReviewer) {
pools = append(pools, "reviewed")
}
if annotationManager(principal) {
pools = append(pools, "all")
}
for _, pool := range pools {
count := 0
cursor := ""
for {
if r.Context().Err() != nil {
renderError(w, 500, r.Context().Err())
return
}
next := r.Clone(context.WithValue(r.Context(), humanPoolReadModeKey{}, humanPoolReadCount))
u := *r.URL
next.URL = &u
query := next.URL.Query()
if pool == "reviewed" {
query.Del("stageId")
query.Del("stageIds")
if stages := query.Get("reviewedStageIds"); stages != "" {
query.Set("stageIds", stages)
}
}
query.Del("reviewedStageIds")
query.Set("pool", pool)
query.Set("limit", "1000")
query.Set("cursor", cursor)
next.URL.RawQuery = query.Encode()
page, readRequest := s.readHumanItemsList(w, next)
if page == nil {
return
}
r = r.WithContext(readRequest.Context())
count += len(page.Items)
if !page.HasMore {
break
}
if page.NextCursor == "" || page.NextCursor == cursor {
renderJSON(w, 500, map[string]string{"error": "任务数量暂时无法核对,请重试。"})
return
}
cursor = page.NextCursor
}
counts[pool] = count
}
renderJSON(w, 200, map[string]any{"counts": counts, "filtersApplied": true})
}

View File

@ -0,0 +1,101 @@
package api
import (
"droboticflow/internal/domain"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"time"
)
func TestHumanPoolSummaryAndCountPreserveAuthorization(t *testing.T) {
f := sharedAcceptanceFixture(t)
f.item.Subject.Payload["hugeDetail"] = strings.Repeat("private draft ", 100000)
if err := f.store.UpsertHumanItem(f.ctx, f.item); err != nil {
t.Fatal(err)
}
full := f.request("GET", "/api/human-items?pluginId=annotation-qa&pool=available", f.reviewer, nil)
summary := f.request("GET", "/api/human-item-summaries?pluginId=annotation-qa&pool=available", f.reviewer, nil)
if full.Code != 200 || summary.Code != 200 {
t.Fatalf("full=%d summary=%d %s", full.Code, summary.Code, summary.Body.String())
}
var before struct {
Items []domain.HumanItemRecord `json:"items"`
}
var after struct {
Summaries []humanPoolSummary `json:"summaries"`
}
if err := json.Unmarshal(full.Body.Bytes(), &before); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(summary.Body.Bytes(), &after); err != nil {
t.Fatal(err)
}
// Compare the public metadata contract independently of the DTO converter.
var fullJSON, summaryJSON map[string]any
_ = json.Unmarshal(full.Body.Bytes(), &fullJSON)
_ = json.Unmarshal(summary.Body.Bytes(), &summaryJSON)
for index, value := range fullJSON["items"].([]any) {
expected := value.(map[string]any)
delete(expected["subject"].(map[string]any), "payload")
actual := summaryJSON["summaries"].([]any)[index].(map[string]any)
delete(actual, "kind")
delete(actual, "durationSeconds")
if !reflect.DeepEqual(expected, actual) {
t.Fatalf("public list metadata changed: expected=%v actual=%v", expected, actual)
}
}
if len(before.Items) != len(after.Summaries) || len(after.Summaries) == 0 {
t.Fatal("summary lost rows")
}
for i, item := range before.Items {
want := summarizeHumanPoolItem(item)
if !reflect.DeepEqual(want, after.Summaries[i]) {
t.Fatalf("summary metadata changed: %#v %#v", want, after.Summaries[i])
}
}
if len(summary.Body.Bytes()) >= 10000 || strings.Contains(summary.Body.String(), "private draft") {
t.Fatal("summary returned detail")
}
counts := f.request("GET", "/api/human-item-counts?pluginId=annotation-qa&stageIds=acceptance,prepackage_acceptance&reviewedStageIds=acceptance,prepackage_acceptance", f.reviewer, nil)
var totals struct {
Counts map[string]int `json:"counts"`
}
if counts.Code != 200 || json.Unmarshal(counts.Body.Bytes(), &totals) != nil || totals.Counts["available"] != len(before.Items) {
t.Fatalf("counts=%d %s", counts.Code, counts.Body.String())
}
if _, err := f.store.SetSupplierAcceptanceAuthorization(f.ctx, f.item.OrganizationID, f.reviewer.SupplierID, false, 1, f.admin.ID); err != nil {
t.Fatal(err)
}
for _, path := range []string{"/api/human-item-summaries?pool=available&pluginId=annotation-qa", "/api/human-item-counts?pluginId=annotation-qa"} {
result := f.request("GET", path, f.reviewer, nil)
if result.Code != 200 || strings.Contains(result.Body.String(), `"canClaim":true`) {
t.Fatalf("revoked result %d %s", result.Code, result.Body.String())
}
}
}
func TestHumanPoolCountsTraversesPagesWithoutHydratingPayloads(t *testing.T) {
f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1)
for n := 0; n < 1005; n++ {
item := f.item
item.ID = fmt.Sprintf("count-%04d", n)
item.CreatedAt = time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339Nano)
item.Subject.Payload = map[string]any{"hugeDetail": strings.Repeat("x", 10000)}
if err := f.store.UpsertHumanItem(f.ctx, item); err != nil {
t.Fatal(err)
}
}
result := f.request("GET", "/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review", f.quality, nil)
var body struct {
Counts map[string]int `json:"counts"`
}
if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &body) != nil || body.Counts["available"] != 1006 {
t.Fatalf("count=%d %s", result.Code, result.Body.String())
}
if result.Body.Len() > 500 {
t.Fatal("count hydrated details")
}
}

View File

@ -84,7 +84,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
if totalActors > 250 {
t.Fatal("local read mix supports at most 250 actors")
}
availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8)
if availablePerStage > 250 {
t.Fatal("available fixtures limited to 250 per stage and supplier")
}
claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1"
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
mode := os.Getenv("POLL_POOL_MODE")
if mode == "" {
@ -214,6 +219,14 @@ func TestPollPoolReadMixLocal(t *testing.T) {
t.Fatal("run payload must be 0..2048 KiB")
}
}
taskPayloadKiB := 0
if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" {
taskPayloadKiB, err = strconv.Atoi(raw)
must(err)
if taskPayloadKiB < 0 || taskPayloadKiB > 2048 {
t.Fatal("task payload must be 0..2048 KiB")
}
}
seededRuns := map[string]bool{}
seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) {
batchID := "batch-" + p + s
@ -237,7 +250,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
taskID := "task-" + id
must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"}))
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}}}))
itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}
if taskPayloadKiB > 0 {
content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024]
itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}}
}
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}}))
}
for _, a := range actors {
if a.Role == "admin" {
@ -269,7 +287,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
stages = append(stages, "annotation_combined")
}
for _, stage := range stages {
for n := 0; n < 8; n++ {
for n := 0; n < availablePerStage; n++ {
seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady)
}
}
@ -335,6 +353,13 @@ func TestPollPoolReadMixLocal(t *testing.T) {
if authorizationState == "initialized" {
must(ds.InitializeSupplierProjectAuthorizations(ctx, org))
}
if readModels {
result, err := ds.BackfillPoolSearchMetadata(ctx, org, true)
must(err)
if result.Unsupported > 0 || result.Conflicts > 0 {
t.Fatalf("summary preparation incomplete: %+v", result)
}
}
handler := NewHandler(ds, nil)
var mu sync.Mutex
samples := []pollLoadSample{}
@ -438,7 +463,11 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
result := map[string]string{}
for _, pool := range []string{"available", "mine", "reviewed"} {
base := "/api/human-items?pluginId=annotation-qa&pool=" + pool + scope
endpoint := "/api/human-items"
if readModels {
endpoint = "/api/human-item-summaries"
}
base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope
if a.Role == "quality" {
result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review"
} else {
@ -447,6 +476,26 @@ func TestPollPoolReadMixLocal(t *testing.T) {
}
}
}
if readModels && countCycle {
stages := "quality_review,semantic_review"
reviewed := ""
if a.Role == "acceptance" {
stages = "acceptance,prepackage_acceptance"
reviewed = "&reviewedStageIds=" + stages
}
result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed
}
if os.Getenv("POLL_POOL_SEARCH") == "1" {
endpoint := "/api/human-items"
if readModels {
endpoint = "/api/human-item-summaries"
}
stage := "quality_review"
if a.Role == "acceptance" {
stage = "prepackage_acceptance"
}
result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
}
return result
}
// Fail before timing if permissions or fixtures accidentally produce empty work.
@ -610,7 +659,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
sort.Float64s(lat)
metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows}
}
report := map[string]any{"schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
report := map[string]any{"read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
encoded, e := json.MarshalIndent(report, "", " ")
must(e)
must(os.WriteFile(out, encoded, 0600))

View File

@ -21,7 +21,7 @@ func productionQualityDates(r *http.Request, now time.Time) (time.Time, time.Tim
if zone == "" {
zone = "Asia/Shanghai"
}
loc, e := time.LoadLocation(zone)
loc, e := workforcestats.ReportingLocation(zone)
if e != nil {
return time.Time{}, time.Time{}, nil, e
}

View File

@ -8,6 +8,8 @@ import "github.com/go-chi/chi/v5"
func (server *server) mountHumanLoopRoutes(router chi.Router) {
router.Get("/api/human-batches", server.handleHumanBatchesList)
router.Get("/api/human-items", server.handleHumanItemsList)
router.Get("/api/human-item-summaries", server.handleHumanPoolSummaries)
router.Get("/api/human-item-counts", server.handleHumanPoolCounts)
router.Get("/api/plugins/annotation-qa/claim-projects", server.handleAnnotationClaimProjects)
router.Get("/api/plugins/annotation-qa/claim-count", server.handleAnnotationClaimCount)
router.Get("/api/human-items/{id}", server.handleHumanItemGet)

View File

@ -1,7 +1,7 @@
# Generated by TestRouteSnapshot; see routes_snapshot_test.go.
# access: reach = endpoint reached; otherwise the status the middleware chain returned.
global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache]
routes=379
routes=381
DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
@ -67,6 +67,8 @@ GET /api/external/object-sync/sessions/{sessionId}/items | mw=[(*server).require
GET /api/external/object-sync/sessions/{sessionId}/status | mw=[(*server).requireObjectSyncFeature-fm,(*server).objectSyncCredentialAuth-fm,(*server).objectSyncDistributedRateLimit.1,(*server).requireObjectSyncIdempotency-fm] | (*server).handleExternalObjectSyncSessionStatus-fm | anon=401 token=401 super=401 admin=401 supplier=401 annotator=401 quality=401 acceptance=401
GET /api/health | mw=[] | (*server).handleHealth-fm | anon=reach token=reach super=reach admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/human-batches | mw=[] | (*server).handleHumanBatchesList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/human-item-counts | mw=[] | (*server).handleHumanPoolCounts-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/human-item-summaries | mw=[] | (*server).handleHumanPoolSummaries-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/human-items | mw=[] | (*server).handleHumanItemsList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/human-items/{id} | mw=[] | (*server).handleHumanItemGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/human-items/{id}/acceptance-misreports | mw=[(*server).requireBusinessRole.1] | (*server).handleAcceptanceMisreports-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=reach acceptance=reach

View File

@ -148,7 +148,22 @@ type HumanTraceSummary struct {
Rejections *int `json:"rejections"`
}
// HumanPoolSearchMetadata contains only source-owned names. It is never public
// task content or a permission decision.
type HumanPoolSearchMetadata struct {
Source map[string]any `bson:"source"`
SourceVersion int `bson:"sourceVersion"`
SourceUpdatedAt string `bson:"sourceUpdatedAt"`
SourceItemKey string `bson:"sourceItemKey"`
SourceArtifactID string `bson:"sourceArtifactId"`
DurationSeconds float64 `bson:"durationSeconds"`
Schema int `bson:"schema"`
Text []string `bson:"text"`
PayloadDirectories []string `bson:"payloadDirectories"`
}
type HumanItemRecord struct {
PoolSearch *HumanPoolSearchMetadata `json:"-" bson:"poolSearch,omitempty"`
SourceValidationStatus string `json:"sourceValidationStatus,omitempty" bson:"-"`
TraceSummary *HumanTraceSummary `json:"traceSummary,omitempty" bson:"-"`
ReviewHistory *HumanReviewHistory `json:"reviewHistory,omitempty" bson:"-"`
@ -497,3 +512,10 @@ type HumanRunDisplay struct {
TaskID string `json:"taskId" bson:"taskId"`
CaptureDirectories []string `json:"captureDirectories" bson:"captureDirectories"`
}
// PoolSearchCurrent also detects an old server writing a record without
// refreshing its derived summary during a rolling upgrade or rollback.
func (item HumanItemRecord) PoolSearchCurrent() bool {
m := item.PoolSearch
return m != nil && m.Schema == 1 && m.SourceVersion == item.Version && m.SourceUpdatedAt == item.UpdatedAt && m.SourceItemKey == item.Subject.ItemKey && m.SourceArtifactID == item.Subject.ArtifactID
}

View File

@ -10,6 +10,7 @@ import (
"encoding/json"
"errors"
"fmt"
"golang.org/x/sync/singleflight"
"io"
"log"
"sort"
@ -18,24 +19,27 @@ import (
)
type Reader struct {
querySlots chan struct{}
backend Backend
source Source
now func() time.Time
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
mu sync.Mutex
closed bool
attempts map[string]time.Time
cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache.
cacheBytes int
reads chan struct{}
queryMemo map[string][]byte
queryMemoBytes int
queryFlights singleflight.Group
querySlots chan struct{}
backend Backend
source Source
now func() time.Time
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
mu sync.Mutex
closed bool
attempts map[string]time.Time
cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache.
cacheBytes int
reads chan struct{}
}
func NewReader(b Backend, source Source) *Reader {
ctx, cancel := context.WithCancel(context.Background())
return &Reader{querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
return &Reader{queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
}
func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() }
func scopeKey(org string, projects []string) string {
@ -48,7 +52,7 @@ func scopeKey(org string, projects []string) string {
// Read returns nil while the initial generation is being built. Existing complete
// generations remain available during refresh. Pins never silently change data.
func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) {
func (r *Reader) resolve(ctx context.Context, org string, projects []string, pin string) (*Generation, Info, error) {
key, now := scopeKey(org, projects), r.now()
state, err := r.backend.OutputSnapshotState(ctx)
if err != nil {
@ -98,6 +102,22 @@ func (r *Reader) Read(ctx context.Context, org string, projects []string, pin st
}
return nil, info, nil
}
info.ID = gen.ID
info.AsOf = gen.AsOf
info.Stale = now.Sub(gen.AsOf) >= RefreshInterval
return gen, info, nil
}
func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) {
gen, info, err := r.resolve(ctx, org, projects, pin)
if err != nil || gen == nil {
return nil, info, err
}
return r.readResolved(ctx, gen, info, pin)
}
func (r *Reader) readResolved(ctx context.Context, gen *Generation, info Info, pin string) (*workforcestats.OutputDataset, Info, error) {
now := r.now()
select {
case r.reads <- struct{}{}:
case <-ctx.Done():
@ -105,9 +125,17 @@ func (r *Reader) Read(ctx context.Context, org string, projects []string, pin st
}
defer func() { <-r.reads }()
data, err := r.load(ctx, *gen)
if err != nil && pin == "" && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge {
gen = head.Previous
data, err = r.load(ctx, *gen)
if err != nil && pin == "" {
state, stateErr := r.backend.OutputSnapshotState(ctx)
if stateErr == nil {
for _, head := range state.Heads {
if head.Current != nil && head.Current.ID == gen.ID && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge {
gen = head.Previous
data, err = r.load(ctx, *gen)
break
}
}
}
}
if err != nil {
return nil, info, ErrUnavailable
@ -297,18 +325,137 @@ func (r *Reader) load(ctx context.Context, g Generation) (*workforcestats.Output
// Query bounds the entire decode + report calculation, not only Mongo reads.
func (r *Reader) Query(ctx context.Context, org string, projects []string, pin string, q workforcestats.OutputQuery) (*workforcestats.OutputReport, Info, error) {
gen, info, err := r.resolve(ctx, org, projects, pin)
if err != nil || gen == nil {
return nil, info, err
}
// Scope permissions are supplied afresh by the handler. The complete query,
// generation and location identity participate in the key, including personal
// and supplier isolation. No permissions or mutable datasets are cached.
q.Now = gen.AsOf
keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)})
key := string(keyData)
select {
case r.querySlots <- struct{}{}:
case <-ctx.Done():
return nil, Info{}, ctx.Err()
return nil, info, ctx.Err()
}
defer func() { <-r.querySlots }()
d, info, err := r.Read(ctx, org, projects, pin)
if err != nil || d == nil {
return nil, info, err
cached := r.memoReport(key)
<-r.querySlots
if cached != nil {
return cached, info, nil
}
result := workforcestats.QueryOutputDataset(*d, q)
return &result, info, nil
type answer struct {
report *workforcestats.OutputReport
info Info
}
result := r.queryFlights.DoChan(key, func() (any, error) {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil, ErrUnavailable
}
r.wg.Add(1)
r.mu.Unlock()
defer r.wg.Done()
buildCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
defer cancel()
select {
case r.querySlots <- struct{}{}:
case <-buildCtx.Done():
return nil, buildCtx.Err()
}
defer func() { <-r.querySlots }()
if report := r.memoReport(key); report != nil {
return answer{report, info}, nil
}
data, actual, err := r.readResolved(buildCtx, gen, info, pin)
if err != nil || data == nil {
return nil, err
}
report := workforcestats.QueryOutputDataset(*data, q)
// Corrupt-current fallback must not poison the current generation's key.
if actual.ID == gen.ID {
r.rememberReport(key, &report)
}
return answer{&report, actual}, nil
})
select {
case <-ctx.Done():
return nil, info, ctx.Err()
case value := <-result:
if value.Err != nil {
return nil, info, value.Err
}
valueAnswer := value.Val.(answer)
select {
case r.querySlots <- struct{}{}:
case <-ctx.Done():
return nil, info, ctx.Err()
}
defer func() { <-r.querySlots }()
// Every caller receives an independent value; UI/report handlers may paginate
// or modify the report without mutating another user's response.
raw, err := json.Marshal(valueAnswer.report)
if err != nil {
return nil, info, err
}
var report workforcestats.OutputReport
if err = json.Unmarshal(raw, &report); err != nil {
return nil, info, err
}
return &report, valueAnswer.info, nil
}
}
const queryMemoBudget = 4 << 20
func (r *Reader) memoReport(key string) *workforcestats.OutputReport {
r.mu.Lock()
payload := r.queryMemo[key]
r.mu.Unlock()
if payload == nil {
return nil
}
reader, err := gzip.NewReader(bytes.NewReader(payload))
if err != nil {
return nil
}
defer reader.Close()
var report workforcestats.OutputReport
if json.NewDecoder(io.LimitReader(reader, MaxDecodedBytes+1)).Decode(&report) != nil {
return nil
}
return &report
}
func (r *Reader) rememberReport(key string, report *workforcestats.OutputReport) {
raw, err := json.Marshal(report)
if err != nil || len(raw) > MaxDecodedBytes {
return
}
var encoded bytes.Buffer
writer, _ := gzip.NewWriterLevel(&encoded, gzip.BestSpeed)
if _, err = writer.Write(raw); err != nil {
return
}
if writer.Close() != nil || encoded.Len() > queryMemoBudget {
return
}
r.mu.Lock()
defer r.mu.Unlock()
if r.queryMemo[key] != nil {
return
}
cost := encoded.Len() + len(key) + 128
if cost > queryMemoBudget {
return
}
if len(r.queryMemo) >= 128 || r.queryMemoBytes+cost > queryMemoBudget {
r.queryMemo = map[string][]byte{}
r.queryMemoBytes = 0
}
r.queryMemo[key] = encoded.Bytes()
r.queryMemoBytes += cost
}
func failureCode(err error) string {

View File

@ -267,3 +267,34 @@ func TestSnapshotScopeCountCannotGrowWithoutBound(t *testing.T) {
t.Fatal("scope limit not enforced")
}
}
func TestReportMemoScopeGenerationAndMutationIsolation(t *testing.T) {
now := time.Now().UTC().Truncate(time.Millisecond)
r := NewReader(&MemoryBackend{}, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
return testData(now), nil
})
defer r.Close()
_, generation := ready(t, r, "")
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
first, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
if err != nil || first == nil {
t.Fatalf("query: %v", err)
}
first.Totals.Submitted.Count = -100
q.Now = now.Add(time.Minute)
second, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
if err != nil || second.Totals.Submitted.Count != 12 {
t.Fatalf("memo mutated: %+v %v", second, err)
}
q.ScopeUserID = "other"
other, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
if err != nil || other.Totals.Submitted.Count != 0 {
t.Fatalf("scope leaked: %+v %v", other, err)
}
r.mu.Lock()
entries, bytes := len(r.queryMemo), r.queryMemoBytes
r.mu.Unlock()
if entries != 2 || bytes > queryMemoBudget {
t.Fatalf("memo entries=%d bytes=%d", entries, bytes)
}
}

View File

@ -169,7 +169,11 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
}
if len(values) == 0 {
values = append(values, domain.CaptureDirectoryFromArtifactValue(row.Subject.ItemKey))
domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values)
if row.PoolSearchCurrent() {
values = append(values, row.PoolSearch.PayloadDirectories...)
} else {
domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values)
}
}
for _, v := range []string{task.ArtifactID, task.SourceVideo} {
values = append(values, domain.CaptureDirectoryFromArtifactValue(v))

View File

@ -50,6 +50,7 @@ func annotationSafeRunUpdate(stored domain.RunRecord, unset []string) (mongo.Pip
if e = bson.Unmarshal(raw, &fields); e != nil {
return nil, e
}
stampPoolSearchMetadata("runs", fields)
set := bson.M{}
for key, value := range fields {
set[key] = bson.M{"$literal": value}

View File

@ -271,6 +271,7 @@ func (store *MongoStore) upsertCaptureSupplierGuarded(ctx context.Context, item
if err != nil {
return err
}
stampPoolSearchMetadata("captureRecords", set)
update := bson.M{"$set": set}
if unset := captureRecordUnsetFields(item); len(unset) > 0 {
for field := range unset {

View File

@ -85,6 +85,12 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
defer cancel()
if err := ctx.Err(); err != nil {
return nil, err
}
if q.IDs != nil && len(q.IDs) == 0 || q.AvailableProjectIDs != nil && len(q.AvailableProjectIDs) == 0 {
return []domain.HumanItemRecord{}, nil
}
filter := humanItemMongoFilter(org, q)
if q.BlindAnnotationActorID != "" {
visibility, err := s.blindAnnotationMongoVisibility(ctx, org, q)
@ -93,6 +99,13 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma
}
filter = bson.M{"$and": bson.A{filter, visibility}}
}
candidates, err := s.humanPoolSearchCandidates(ctx, org, q, filter)
if err != nil {
return nil, err
}
if candidates != nil {
filter = bson.M{"$and": bson.A{filter, candidates}}
}
pipeline := humanPoolSearchPipeline(org, q, filter)
for _, text := range []string{q.Search, q.Capture} {
if text != "" {

View File

@ -13,7 +13,7 @@ import (
func humanPoolSearchPipeline(org string, q domain.HumanItemQuery, filter bson.M) mongo.Pipeline {
// Only search/category metadata participates in global sorts and joins.
// Hydrate matched IDs afterwards; the public record shape stays unchanged.
projection := bson.M{"_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1}
projection := bson.M{"version": 1, "updatedAt": 1, "poolSearch": 1, "_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1}
pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: projection}}}
pipeline = append(pipeline,
// Number candidate batches; restore the stable createdAt/id order below.

View File

@ -24,7 +24,7 @@ func (s *MongoStore) humanPoolSearchLegacyPayloads(ctx context.Context, org stri
name = domain.CaptureDirectoryFromDirectoryValue(row.Captures[0].Prefix)
}
}
if name == "" {
if name == "" && !row.PoolSearchCurrent() {
ids = append(ids, row.ID)
}
}

View File

@ -25,7 +25,10 @@ func TestHumanPoolSearchMongoLargeModern(t *testing.T) {
func TestHumanPoolSearchMongoLargePayload(t *testing.T) { testHumanPoolSearchMongo(t, 420, true, true) }
func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) {
func TestHumanPoolSearchMongoReadModel(t *testing.T) {
testHumanPoolSearchMongo(t, 10000, true, false, true)
}
func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool, migrate ...bool) {
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
if uri == "" {
t.Skip("isolated local Mongo opt-in")
@ -197,6 +200,20 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) {
if _, err := s.annotationTasks.InsertOne(ctx, foreignTask); err != nil {
t.Fatal(err)
}
if len(migrate) > 0 && migrate[0] {
dry, err := s.BackfillPoolSearchMetadata(ctx, org, false)
if err != nil || dry.Scanned == 0 || dry.Updated != 0 {
t.Fatalf("dry run: %+v %v", dry, err)
}
result, err := s.BackfillPoolSearchMetadata(ctx, org, true)
if err != nil || result.Updated != dry.Scanned || result.Conflicts != 0 || result.Unsupported != 0 {
t.Fatalf("backfill: %+v %v", result, err)
}
again, err := s.BackfillPoolSearchMetadata(ctx, org, true)
if err != nil || again.Scanned != 0 {
t.Fatalf("idempotence: %+v %v", again, err)
}
}
ids := func(t *testing.T, rows []domain.HumanItemRecord) []string {
out := []string{}
for _, i := range rows {
@ -281,4 +298,47 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) {
}
})
}
if len(migrate) > 0 && migrate[0] {
item, err := s.GetHumanItem(ctx, "i-0008")
if err != nil {
t.Fatal(err)
}
item.Version++
item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
if _, err = s.humanItems.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": bson.M{"subject.payload.importManifest.sourceDirectory": "archive/moved-legacy-capture", "version": item.Version, "updatedAt": item.UpdatedAt}}); err != nil {
t.Fatal(err)
}
got, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: "moved-legacy-capture", Limit: 200})
if err != nil || len(got) != 1 || got[0].ID != item.ID {
t.Fatalf("old writer compatibility: %v %v", ids(t, got), err)
}
result, err := s.BackfillPoolSearchMetadata(ctx, org, true)
if err != nil || result.Updated != 1 {
t.Fatalf("repair stale summary: %+v %v", result, err)
}
current, err := s.GetHumanItem(ctx, item.ID)
if err != nil || !current.PoolSearchCurrent() {
t.Fatalf("repaired summary not current: %v", err)
}
next := *current
next.Version++
next.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
next.Status = domain.HumanItemAccepted
changed, err := s.UpdateHumanItemForReview(ctx, next, current.Version)
if err != nil || !changed {
t.Fatalf("review update: %v %v", changed, err)
}
current, err = s.GetHumanItem(ctx, item.ID)
if err != nil || !current.PoolSearchCurrent() || current.Subject.Payload["importManifest"] == nil {
t.Fatalf("review lost summary or annotation: %v", err)
}
if _, err = s.projects.UpdateOne(ctx, bson.M{"id": "p"}, bson.M{"$set": bson.M{"name": "renamed-project"}}); err != nil {
t.Fatal(err)
}
got, err = s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: "renamed-project", Limit: 200, IdentityProjection: true})
if err != nil || len(got) != 200 {
t.Fatalf("legacy project rename: count=%d error=%v", len(got), err)
}
}
}

View File

@ -529,6 +529,7 @@ func (store *MongoStore) UpsertProject(ctx context.Context, item domain.ProjectR
return err
}
persisted := projectPersistenceDocument(item)
stampPoolSearchMetadata("projects", persisted)
_, err = store.projects.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": persisted}, options.Update().SetUpsert(allow))
return err
}

View File

@ -0,0 +1,61 @@
package store
import (
"context"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
type PoolSearchBackfillResult struct{ Scanned, Updated, Conflicts, Unsupported int64 }
// BackfillPoolSearchMetadata is an explicit maintenance operation, never called
// from GETs or startup. A full-document CAS prevents overwriting any concurrent
// writer, including legacy writers without versions/timestamps. Retry conflicts
// by rerunning; successful records are skipped. No indexes or business fields
// are changed. Dry runs are read-only.
func (s *MongoStore) BackfillPoolSearchMetadata(ctx context.Context, org string, apply bool) (PoolSearchBackfillResult, error) {
result := PoolSearchBackfillResult{}
for _, name := range []string{"humanItems", "runs", "annotationTasks", "captureRecords", "projects"} {
collection := s.db.Collection(name)
filter := bson.M{"organizationId": normalizedOrganizationID(org), "$expr": bson.M{"$not": bson.A{poolSearchValidExpression(name)}}}
cursor, err := collection.Find(ctx, filter, options.Find().SetBatchSize(16))
if err != nil {
return result, err
}
err = func() error {
defer cursor.Close(ctx)
for cursor.Next(ctx) {
original := append(bson.Raw(nil), cursor.Current...)
var d bson.M
if err := bson.Unmarshal(original, &d); err != nil {
return err
}
result.Scanned++
stampPoolSearchMetadata(name, d)
metadata := d[poolSearchField]
if metadata == nil {
result.Unsupported++
continue
}
if !apply {
continue
}
update, err := collection.UpdateOne(ctx, bson.M{"_id": d["_id"], "$expr": bson.M{"$eq": bson.A{"$$ROOT", bson.M{"$literal": original}}}}, bson.M{"$set": bson.M{poolSearchField: metadata}})
if err != nil {
return fmt.Errorf("backfill %s: %w", name, err)
}
if update.MatchedCount == 0 {
result.Conflicts++
} else {
result.Updated++
}
}
return cursor.Err()
}()
if err != nil {
return result, err
}
}
return result, nil
}

View File

@ -0,0 +1,330 @@
package store
import (
"context"
"droboticflow/internal/domain"
"fmt"
"math"
"regexp"
"sort"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// Owned by the source document, updated in the same write after supplier
// scrubbing. This contains names only, never permissions, eligibility decisions or customer text.
const poolSearchSchema = 1
const poolSearchField = "poolSearch"
type poolSearchMetadata = domain.HumanPoolSearchMetadata
func stampPoolSearchMetadata(collection string, d bson.M) {
m := poolSearchMetadata{Source: map[string]any{}, Schema: poolSearchSchema, Text: []string{}, PayloadDirectories: []string{}}
add := func(keys ...string) {
for _, k := range keys {
if v, ok := d[k].(string); ok {
m.Text = append(m.Text, v)
}
}
}
switch collection {
case "humanItems":
add("id", "legacyTaskId")
subject := adminPoolMap(d["subject"])
for _, k := range []string{"itemKey", "artifactId"} {
if v, ok := subject[k].(string); ok {
m.Text = append(m.Text, v)
}
}
domain.AppendCaptureDirectoriesFromPayload(subject["payload"], &m.PayloadDirectories)
for _, value := range deletionArray(deletionDoc(subject["payload"])["items"]) {
coverage := deletionDoc(deletionDoc(deletionDoc(value)["materialization"])["coverage"])
var seconds float64
switch n := coverage["durationSeconds"].(type) {
case float64:
seconds = n
case int:
seconds = float64(n)
case int32:
seconds = float64(n)
case int64:
seconds = float64(n)
}
if seconds > 0 && !math.IsInf(seconds, 0) && !math.IsNaN(seconds) {
m.DurationSeconds += seconds
}
}
m.Text = append(m.Text, m.PayloadDirectories...)
case "annotationTasks":
add("artifactId", "sourceVideo")
case "captureRecords":
add("captureId", "prefix")
case "projects":
add("name")
case "runs":
// Decode only the fields used by the existing directory parser.
raw, err := bson.Marshal(poolRunSearchDocument(d))
var run domain.RunRecord
if err != nil || bson.Unmarshal(raw, &run) != nil {
d[poolSearchField] = nil
return
}
m.Text = append(m.Text, domain.CaptureDirectoriesFromRun(run)...)
add("projectName")
default:
return
}
normalize := func(values []string) []string {
seen := map[string]bool{}
out := []string{}
for _, v := range values {
v = strings.ToLower(strings.TrimSpace(v))
if v != "" && !seen[v] {
seen[v] = true
out = append(out, v)
}
}
sort.Strings(out)
return out
}
for _, value := range append([]string(nil), m.Text...) {
m.Text = append(m.Text, domain.CaptureDirectoryFromArtifactValue(value), domain.CaptureDirectoryFromDirectoryValue(value))
}
m.Text = normalize(m.Text)
for n, path := range poolSearchSourceFields(collection, d) {
m.Source[fmt.Sprint(n)] = poolSearchSourceValue(d, path)
}
if collection == "humanItems" {
m.SourceVersion = int(supplierNumber(d["version"]))
m.SourceUpdatedAt, _ = d["updatedAt"].(string)
subject := adminPoolMap(d["subject"])
m.SourceItemKey, _ = subject["itemKey"].(string)
m.SourceArtifactID, _ = subject["artifactId"].(string)
}
// Preserve the exact historical values/order for the final matcher. Only
// prefilter text is normalized; the matcher owns display precedence.
encoded, err := bson.Marshal(m)
if err != nil || len(encoded) > 64<<10 {
d[poolSearchField] = nil
return
}
d[poolSearchField] = m
}
// A conservative reverse lookup narrows candidates BEFORE joins/sorts. Missing
// summaries always enter the old path; the final matcher and live permissions
// remain authoritative. Very broad reference sets use the bounded old pipeline
// rather than truncating a result or building an unbounded Mongo command.
func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, q domain.HumanItemQuery, base bson.M) (bson.M, error) {
text := q.Capture
if text == "" {
text = q.Search
}
words := strings.Fields(text)
if len(words) == 0 {
return nil, nil
}
// Do not fetch thousands of foreign legacy IDs just to discover that the
// original pipeline is required. A tiny readiness probe preserves the legacy
// no-hit wire budget and keeps mixed-version reads on the proven path.
legacy := bson.M{"$and": bson.A{base, bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression("humanItems")}}}}}
err := s.humanItems.FindOne(ctx, legacy, options.FindOne().SetProjection(bson.M{"_id": 1}).SetMaxTime(8*time.Second)).Err()
if err == nil {
return nil, nil
}
if err != mongo.ErrNoDocuments {
return nil, err
}
token := strings.ToLower(words[0])
if i := strings.LastIndex(token, "/part"); i > 0 {
token = token[:i]
}
pattern := regexp.QuoteMeta(token) + "|%[0-9a-f]{2}"
matchFor := func(collection string) bson.M {
return bson.M{"$or": bson.A{bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression(collection)}}}, bson.M{poolSearchField + ".text": bson.M{"$regex": pattern}}}}
}
match := matchFor("captureRecords")
const maxReferences = 4096
ids := func(collection string, extra bson.M) ([]string, bool, error) {
f := bson.M{"organizationId": normalizedOrganizationID(org), "$and": bson.A{match}}
if extra != nil {
f["$and"] = append(f["$and"].(bson.A), extra)
}
cur, err := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetLimit(maxReferences+1).SetMaxTime(8*time.Second))
if err != nil {
return nil, false, err
}
defer cur.Close(ctx)
out := []string{}
for cur.Next(ctx) {
var v struct {
ID string `bson:"id"`
}
if err := cur.Decode(&v); err != nil {
return nil, false, err
}
out = append(out, v.ID)
}
return out, len(out) > maxReferences, cur.Err()
}
captureIDs, broad, err := ids("captureRecords", nil)
if err != nil || broad {
return nil, err
}
// A capture match follows only same-organization runs. Tenant checks in the
// existing join are retained even after this candidate narrowing.
runMatch := matchFor("runs")
if len(captureIDs) > 0 {
runMatch = bson.M{"$or": bson.A{runMatch, bson.M{"taskId": bson.M{"$in": captureIDs}}}}
}
match = runMatch
runIDs, broad, err := ids("runs", nil)
if err != nil || broad {
return nil, err
}
match = matchFor("annotationTasks")
taskIDs, broad, err := ids("annotationTasks", nil)
if err != nil || broad {
return nil, err
}
match = matchFor("projects")
projectIDs, broad, err := ids("projects", nil)
if err != nil || broad {
return nil, err
}
branches := bson.A{matchFor("humanItems")}
for field, values := range map[string][]string{"runId": runIDs, "legacyTaskId": taskIDs, "projectId": projectIDs} {
if len(values) > 0 {
branches = append(branches, bson.M{field: bson.M{"$in": values}})
}
}
return bson.M{"$or": branches}, nil
}
func poolSearchSourceValue(d bson.M, path string) any {
if path == "__runReferences" {
return poolRunReferences(d)
}
parts := strings.Split(path, ".")
for _, part := range parts[:len(parts)-1] {
d = deletionDoc(d[part])
}
return d[parts[len(parts)-1]]
}
func poolSearchSourceFields(collection string, d bson.M) []string {
switch collection {
case "humanItems":
return []string{"version", "updatedAt", "subject.itemKey", "subject.artifactId"}
case "projects":
return []string{"name"}
case "annotationTasks":
return []string{"artifactId", "sourceVideo"}
case "captureRecords":
return []string{"captureId", "prefix"}
case "runs":
return []string{"projectName", "captureDirectories", "__runReferences"}
}
return nil
}
func poolSearchValidExpression(collection string) bson.M {
fields := poolSearchSourceFields(collection, nil)
terms := bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$poolSearch.schema", 0}}, poolSearchSchema}}}
for n, path := range fields {
terms = append(terms, bson.M{"$eq": bson.A{poolSearchSourceExpression(path), bson.M{"$ifNull": bson.A{"$poolSearch.source." + fmt.Sprint(n), nil}}}})
}
return bson.M{"$and": terms}
}
func poolSearchSourceExpression(path string) any {
if path != "__runReferences" {
return bson.M{"$ifNull": bson.A{"$" + path, nil}}
}
text := func(value string) bson.M { return bson.M{"$ifNull": bson.A{value, ""}} }
names := func(input string) bson.M {
return bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "as": "artifact", "in": bson.A{text("$$artifact.id"), text("$$artifact.name"), text("$$artifact.path")}}}
}
inputs := bson.M{"$map": bson.M{"input": bson.M{"$sortArray": bson.M{"input": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$inputs", bson.M{}}}}, "sortBy": bson.M{"k": 1}}}, "as": "input", "in": bson.A{"$$input.k", text("$$input.v.ref.captureUnit"), text("$$input.v.ref.prefix"), text("$$input.v.ref.key"), text("$$input.v.ref.manifestKey")}}}
nodes := bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "node", "in": names("$$node.artifacts")}}
return bson.M{"$cond": bson.A{bson.M{"$gt": bson.A{bson.M{"$size": bson.M{"$ifNull": bson.A{"$captureDirectories", bson.A{}}}}, 0}}, bson.A{}, bson.A{inputs, names("$artifacts"), nodes}}}
}
func poolRunReferences(d bson.M) []any {
if len(deletionArray(d["captureDirectories"])) > 0 {
return []any{}
}
text := func(d bson.M, key string) string { v, _ := d[key].(string); return v }
names := func(value any) []any {
out := []any{}
for _, v := range deletionArray(value) {
a := deletionDoc(v)
out = append(out, []any{text(a, "id"), text(a, "name"), text(a, "path")})
}
return out
}
inputs := deletionDoc(d["inputs"])
keys := []string{}
for k := range inputs {
keys = append(keys, k)
}
sort.Strings(keys)
refs := []any{}
for _, k := range keys {
ref := deletionDoc(deletionDoc(inputs[k])["ref"])
refs = append(refs, []any{k, text(ref, "captureUnit"), text(ref, "prefix"), text(ref, "key"), text(ref, "manifestKey")})
}
nodes := []any{}
for _, v := range deletionArray(d["nodeExecutions"]) {
nodes = append(nodes, names(deletionDoc(v)["artifacts"]))
}
return []any{refs, names(d["artifacts"]), nodes}
}
func poolRunSearchDocument(d bson.M) bson.M {
names := func(value any) []any {
out := []any{}
for _, value := range deletionArray(value) {
row := deletionDoc(value)
out = append(out, bson.M{"id": row["id"], "name": row["name"], "path": row["path"]})
}
return out
}
inputs := bson.M{}
for key, value := range deletionDoc(d["inputs"]) {
ref := deletionDoc(deletionDoc(value)["ref"])
selected := bson.M{}
for _, field := range []string{"captureUnit", "prefix", "key", "manifestKey"} {
if v, ok := ref[field]; ok {
selected[field] = v
}
}
inputs[key] = bson.M{"ref": selected}
}
nodes := []any{}
for _, value := range deletionArray(d["nodeExecutions"]) {
nodes = append(nodes, bson.M{"artifacts": names(deletionDoc(value)["artifacts"])})
}
return bson.M{"captureDirectories": d["captureDirectories"], "inputs": inputs, "artifacts": names(d["artifacts"]), "nodeExecutions": nodes}
}
// State-only review writes must keep a valid name summary current without
// re-reading/rewriting customer annotation payloads. Invalid summaries remain
// invalid and take the compatibility path. The version CAS fences this update.
func poolSearchStateUpdate(fields bson.M) mongo.Pipeline {
set := bson.M{}
for key, value := range fields {
set[key] = bson.M{"$literal": value}
}
set[poolSearchField] = bson.M{"$cond": bson.A{
poolSearchValidExpression("humanItems"),
bson.M{"$mergeObjects": bson.A{"$poolSearch", bson.M{
"sourceVersion": bson.M{"$literal": fields["version"]},
"sourceUpdatedAt": bson.M{"$literal": fields["updatedAt"]},
"source": bson.M{"$mergeObjects": bson.A{"$poolSearch.source", bson.M{"0": bson.M{"$literal": fields["version"]}, "1": bson.M{"$literal": fields["updatedAt"]}}}},
}}},
"$poolSearch",
}}
return mongo.Pipeline{bson.D{{Key: "$set", Value: set}}}
}

View File

@ -0,0 +1,43 @@
package store
import (
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"reflect"
"testing"
)
func TestPoolSearchMetadataOwnsOnlyNames(t *testing.T) {
doc := supplierDoc(domain.HumanItemRecord{ID: "i", Subject: domain.HumanSubjectRef{ItemKey: "archive/目录%20甲/part0002.mp4", Payload: map[string]any{"private": "secret comment", "nested": map[string]any{"sourceDirectory": "archive/old-capture"}}}})
stampPoolSearchMetadata("humanItems", doc)
m := doc[poolSearchField].(poolSearchMetadata)
if !reflect.DeepEqual(m.PayloadDirectories, []string{"old-capture"}) {
t.Fatal(m)
}
for _, v := range m.Text {
if v == "secret comment" {
t.Fatal("customer text indexed")
}
}
found := false
for _, v := range m.Text {
if v == "archive/目录%20甲/part0002.mp4" {
found = true
}
}
if !found {
t.Fatal("encoded artifact compatibility", m)
}
row := adminPoolRow{HumanItemRecord: domain.HumanItemRecord{PoolSearch: &m, Subject: domain.HumanSubjectRef{ItemKey: m.SourceItemKey}}}
if !humanPoolSearchRowMatches(row, domain.HumanItemQuery{Capture: "old-capture"}) {
t.Fatal("summary lost legacy directory")
}
// Authoritative capture names keep their precedence over old payload names.
row.Captures = []domain.CaptureRecord{{CaptureID: "authoritative"}}
if humanPoolSearchRowMatches(row, domain.HumanItemQuery{Capture: "old-capture"}) {
t.Fatal("payload overrode capture")
}
if _, err := bson.Marshal(doc); err != nil {
t.Fatal(err)
}
}

View File

@ -95,7 +95,7 @@ func humanReviewFields(v domain.HumanItemRecord) bson.M {
"reviewOperationId": v.ReviewOperationID}
}
func (s *MongoStore) UpdateHumanItemForReview(ctx context.Context, next domain.HumanItemRecord, expected int) (bool, error) {
r, err := s.humanItems.UpdateOne(ctx, bson.M{"id": next.ID, "organizationId": next.OrganizationID, "version": expected}, bson.M{"$set": humanReviewFields(next)})
r, err := s.humanItems.UpdateOne(ctx, bson.M{"id": next.ID, "organizationId": next.OrganizationID, "version": expected}, poolSearchStateUpdate(humanReviewFields(next)))
return err == nil && r.MatchedCount == 1, err
}
func (s *MemoryStore) UpdateHumanItemForReview(_ context.Context, next domain.HumanItemRecord, expected int) (bool, error) {

View File

@ -162,12 +162,18 @@ func scrubSupplierDocument(d bson.M, p domain.SupplierDeletionPlan) bool {
delete(d, "vendorId")
}
if changed {
delete(d, poolSearchField)
d["updatedAt"] = nowISO()
}
return changed
}
func supplierGuardedMongoWrite(ctx context.Context, col *mongo.Collection, id string, item any, write func(context.Context, bson.M) error) error {
d := supplierDoc(item)
writeSource := write
write = func(ctx context.Context, document bson.M) error {
stampPoolSearchMetadata(col.Name(), document)
return writeSource(ctx, document)
}
if !supplierGuardRelevant(col.Name()) {
return write(ctx, d)
}

View File

@ -0,0 +1,27 @@
package workforcestats
import (
"sync"
"time"
)
var reportLocations = struct {
sync.Mutex
values map[string]*time.Location
}{values: map[string]*time.Location{}}
// ReportingLocation shares immutable IANA rules, with bounded retention. This
// makes location identity stable across equivalent report queries without ever
// treating two arbitrary custom time.Location values as equivalent.
func ReportingLocation(name string) (*time.Location, error) {
reportLocations.Lock()
defer reportLocations.Unlock()
if location := reportLocations.values[name]; location != nil {
return location, nil
}
location, err := time.LoadLocation(name)
if err == nil && len(reportLocations.values) < 32 {
reportLocations.values[name] = location
}
return location, err
}

View File

@ -434,6 +434,53 @@
"tests": true
}
]
},
{
"name": "query-readmodels",
"paths": [
"^internal/store/pool_search.*\\.go$",
"^cmd/pool-search-backfill/",
"^internal/store/(mongo|annotation_artifact|capture_automation|supplier_deletion_guard|review_decision)\\.go$",
"^internal/api/(human_display|production_quality|routes_human_loop)\\.go$",
"^internal/api/testdata/routes\\.golden$",
"^internal/api/poll_pool_load_test\\.go$"
],
"frontend": [],
"go": {
"./internal/api": ["^TestHumanPoolSummary", "^TestHumanPoolCounts", "^TestHumanDisplay", "^TestRouteSnapshot$"],
"./internal/store": ["^TestPoolSearchMetadata"]
},
"task": [
{
"argv": ["npm", "run", "test:go:gate"],
"tests": true
},
{
"argv": ["go", "test", "./cmd/pool-search-backfill", "-count=1"]
},
{
"argv": [
"go",
"test",
"./internal/api",
"-run",
"^TestPollPoolReadMixLocal$",
"-count=1",
"-timeout=3m",
"-v"
],
"env": {
"POLL_POOL_LOAD": "1",
"POLL_POOL_READMODELS": "1",
"POLL_POOL_SEARCH": "1",
"POLL_POOL_AUTHORIZATION_STATE": "initialized",
"POLL_POOL_WARMUP_SECONDS": "5",
"POLL_POOL_DURATION_SECONDS": "30"
},
"tests": true
}
],
"warnings": ["本地 81 人读取冒烟不代表持续容量验收;摘要补建生产 apply 需单独授权,旧记录兼容路径须保留。"]
}
]
}

View File

@ -1,3 +1,4 @@
import { isHumanPoolSummary } from "../types";
import { ReviewListTrace } from "./ReviewTrace";
import type { HumanItem } from "../types";
import { ReviewHistorySummary } from "./ReviewHistorySummary";
@ -32,10 +33,13 @@ export function AcceptanceTaskCard({
const completed = ["accepted", "released"].includes(item.status);
const reworkReview = !completed && item.status !== "rejected" && Number(item.reviewRound || 0) > 0;
const entries = Array.isArray(item.subject?.payload?.items) ? item.subject.payload.items : [];
const seconds = entries.reduce(
(total: number, entry: any) => total + Math.max(0, Number(entry.materialization?.coverage?.durationSeconds) || 0),
0,
);
const seconds = isHumanPoolSummary(item)
? item.durationSeconds || 0
: entries.reduce(
(total: number, entry: any) =>
total + Math.max(0, Number(entry.materialization?.coverage?.durationSeconds) || 0),
0,
);
const action = acceptanceTaskAction({ status: item.status, mine, claimable });
return (
<article

View File

@ -1,3 +1,18 @@
// Only route absence permits a rolling-upgrade fallback. Summary objects stay
// marked as summaries; selected workbenches fetch detail independently.
export async function loadHumanPoolList(human, filters, options) {
if (human.listSummariesPage) {
try {
const page = await human.listSummariesPage(filters, options);
if (!Array.isArray(page.summaries)) throw new Error("任务列表响应不完整,请重试。");
return { ...page, items: page.summaries };
} catch (error) {
if (error?.status !== 404 && error?.status !== 405) throw error;
}
}
return human.listItemsPage(filters, options);
}
// Keep GET joins below proxy request-line limits, with bounded concurrency.
export async function loadAnnotationTaskJoin(annotation, filters, ids, signal = undefined) {
signal?.throwIfAborted();
@ -131,7 +146,8 @@ export async function loadAnnotationPoolPage({ annotation, human, scope, view, c
: ["available", "mine", "reviewed"];
const pages = await Promise.all(
names.map(async (pool) => {
const page = await human.listItemsPage(
const page = await loadHumanPoolList(
human,
{
pluginId: "annotation-qa",
projectId: scope.projectId,

View File

@ -0,0 +1,11 @@
// List summaries select an identity, never editable content. Keep legacy
// annotation/package detail fallback when no aggregate pool item is selected.
export function reviewItemForSelection(poolItem, aggregateDetail, annotationItem) {
if (!poolItem) return annotationItem;
if (poolItem.kind !== "pool-summary") return poolItem;
return aggregateDetail?.id === poolItem.id &&
aggregateDetail.version >= poolItem.version &&
aggregateDetail.kind !== "pool-summary"
? aggregateDetail
: undefined;
}

View File

@ -1,3 +1,5 @@
import { reviewItemForSelection } from "../lib/review-item-selection.js";
import { isHumanPoolSummary } from "../types";
import { annotationTimelineFailure } from "../lib/annotation-source-validation.js";
import type { AnnotationTimelineFailure } from "../lib/annotation-source-validation";
// Keep the workbench rules ahead of the review framing rules, matching the
@ -749,6 +751,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
const [templates, setTemplates] = useState<AnnotationTemplate[]>([]);
const [tasks, setTasks] = useState<AnnotationTaskRecord[]>([]);
const [humanPools, setHumanPools] = useState<HumanPools>(emptyHumanPools);
const [reviewPoolTotals, setReviewPoolTotals] = useState<{
scope: string;
counts: Record<TaskPoolView, number>;
} | null>(null);
const [reviewPoolCountError, setReviewPoolCountError] = useState("");
const [acceptancePoolDetail, setAcceptancePoolDetail] = useState<{ scope: string; item: HumanItem } | null>(null);
const [acceptancePoolDetailError, setAcceptancePoolDetailError] = useState("");
const [acceptanceDetailRetry, setAcceptanceDetailRetry] = useState(0);
// A URL task id is a selection, not ownership; deep-link previews retain their pool.
const defaultTaskPoolView = () =>
initialAnnotationPool({
@ -1065,6 +1075,39 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
...(managerAnnotationPool ? [managerPoolFilters] : []),
...(pagedReviewPool ? [adminSearch, taskStatusFilter, requestedCaptureDirectory] : []),
]);
useEffect(() => {
if (!pagedReviewPool) return;
const controller = new AbortController();
setReviewPoolCountError("");
void humanLoopApi
.poolCounts(
{
pluginId: "annotation-qa",
projectId: projectId || undefined,
workflowId: requestedWorkflowId || undefined,
runId: requestedRunId || undefined,
search: adminSearch || undefined,
capture: requestedCaptureDirectory || undefined,
category: taskStatusFilter,
stageIds: acceptanceWorkspace
? ["acceptance", "prepackage_acceptance"]
: humanItemStageIdsForMode(workspaceMode),
reviewedStageIds: acceptanceWorkspace ? ["acceptance", "prepackage_acceptance"] : undefined,
},
{ signal: controller.signal },
)
.then((result) => {
if (!controller.signal.aborted) setReviewPoolTotals({ scope: projectDataScope, counts: result.counts });
})
.catch((cause) => {
if (!controller.signal.aborted) {
setReviewPoolTotals(null);
if (cause?.status !== 404 && cause?.status !== 405)
setReviewPoolCountError(getActionErrorMessage(cause, "加载任务数量失败"));
}
});
return () => controller.abort();
}, [pagedReviewPool, projectDataScope, humanPools]);
const reviewPoolViewRef = useRef(taskPoolView);
reviewPoolViewRef.current = taskPoolView;
const reviewActionScopeRef = useRef(projectDataScope);
@ -2137,8 +2180,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
if (prepackageAcceptanceItems.some((item) => item.id === requestedAcceptanceId))
setExpandedDeliveryPackageId(requestedAcceptanceId);
}, [workspaceMode, requestedAcceptanceId, prepackageAcceptanceItems, loading, saving]);
const reviewSessionItem =
prepackageAcceptanceItems.find((item) => item.id === expandedDeliveryPackageId) || detail?.humanItem;
const acceptanceListSelection = prepackageAcceptanceItems.find((item) => item.id === expandedDeliveryPackageId);
const acceptanceDetailSelection =
acceptancePoolDetail?.scope === projectDataScope &&
acceptancePoolDetail.item.id === acceptanceListSelection?.id &&
acceptancePoolDetail.item.version >= acceptanceListSelection.version
? acceptancePoolDetail.item
: null;
useEffect(() => {
if (!acceptanceListSelection || !isHumanPoolSummary(acceptanceListSelection)) return;
const controller = new AbortController();
setAcceptancePoolDetailError("");
void humanLoopApi
.getItem(acceptanceListSelection.id, { signal: controller.signal })
.then(({ item }) => {
if (!controller.signal.aborted) setAcceptancePoolDetail({ scope: projectDataScope, item });
})
.catch((cause) => {
if (!controller.signal.aborted) setAcceptancePoolDetailError(getActionErrorMessage(cause, "加载验收内容失败"));
});
return () => controller.abort();
}, [projectDataScope, acceptanceListSelection?.id, acceptanceListSelection?.version, acceptanceDetailRetry]);
const reviewSessionItem = reviewItemForSelection(
acceptanceListSelection,
acceptanceDetailSelection,
detail?.humanItem,
);
const reviewSessionEnabled = Boolean(
(workspaceMode === "quality" || workspaceMode === "clip-quality" || workspaceMode === "acceptance") &&
taskPoolView !== "reviewed" &&
@ -2817,8 +2884,18 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
: { available: 0, mine: 0, all: 0 }),
reviewed: 0,
}
: poolTabCounts(humanPools, matchesPoolScope),
[humanPools, matchesPoolScope, managerAnnotationPool, adminPoolTotals, projectDataScope],
: pagedReviewPool && reviewPoolTotals?.scope === projectDataScope
? reviewPoolTotals.counts
: poolTabCounts(humanPools, matchesPoolScope),
[
humanPools,
matchesPoolScope,
managerAnnotationPool,
adminPoolTotals,
projectDataScope,
pagedReviewPool,
reviewPoolTotals,
],
);
const stageEmptyLabel =
workspaceMode === "quality"
@ -7043,9 +7120,9 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
aria-pressed={taskPoolView === value}
className={`rounded-lg border px-3 py-2 text-xs transition-colors ${taskPoolView === value ? "border-blue-500 bg-blue-50 text-blue-600 dark:bg-blue-500/10 dark:text-blue-200 font-medium" : "border-[color:var(--app-border)] text-slate-600 hover:text-slate-900 dark:text-gray-400 dark:hover:text-white"}`}
>
{label} {pagedReviewPool ? "已加载 " : ""}
{label}
{(managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope) ||
(pagedReviewPool && !(value in reviewPoolCursorsRef.current))
(pagedReviewPool && reviewPoolTotals?.scope !== projectDataScope)
? "—"
: blindAnnotator && value === "available"
? claimableCount.projectId === projectId
@ -7053,10 +7130,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
: "—"
: count}
{/* Counts only loaded pages; a further page means more exist. */}
{(pagedReviewPool && reviewPoolCursorsRef.current[value]) ||
(blindAnnotator && value === "mine" && minePoolCursorRef.current)
? "+"
: ""}
{blindAnnotator && value === "mine" && minePoolCursorRef.current ? "+" : ""}
</button>
))}
</div>
@ -7183,6 +7257,11 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
<div className="annotation-task-list-label mt-4 text-xs font-medium text-slate-500 dark:text-gray-400">
逐段任务
</div>
{reviewPoolCountError && (
<div role="alert" className="mt-2 text-xs text-red-600">
{reviewPoolCountError}
</div>
)}
<div className="annotation-task-list mt-3 max-h-[calc(100vh-330px)] space-y-1.5 overflow-auto pr-1">
{sidebarLoading && (
<div role="status" className="p-3 text-sm">
@ -7582,7 +7661,27 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
)}
{prepackageAcceptanceItems
.filter((i) => i.id === expandedDeliveryPackageId)
.map((item) => {
.map((poolItem) => {
const item = isHumanPoolSummary(poolItem) ? acceptanceDetailSelection : poolItem;
if (!item)
return (
<div
key={poolItem.id}
role={acceptancePoolDetailError ? "alert" : "status"}
className="p-3 text-sm"
>
{acceptancePoolDetailError || "正在加载验收内容…"}
{acceptancePoolDetailError && (
<button
type="button"
className="ml-2 underline"
onClick={() => setAcceptanceDetailRetry((value) => value + 1)}
>
重试
</button>
)}
</div>
);
const stats = prepackageAcceptanceDrafts[item.id] || {
...emptyPrepackageAcceptanceStatistics,
headpose: "include",

View File

@ -1,4 +1,10 @@
import type { HumanItem, HumanItemStatus, HumanReject, RunArtifactManifestMembersResponse } from "../../types";
import type {
HumanItem,
HumanPoolSummary,
HumanItemStatus,
HumanReject,
RunArtifactManifestMembersResponse,
} from "../../types";
import { mergePagedManifestMembers } from "../../lib/annotation-workbench.js";
import { apiRequest } from "./client";
@ -89,6 +95,19 @@ async function listAcceptancePool(filters: HumanItemFilters, options: { signal?:
export const humanLoopApi = {
listItems,
listItemsPage,
listSummariesPage: (filters: HumanItemFilters = {}, options: { signal?: AbortSignal } = {}) =>
apiRequest<{ summaries: HumanPoolSummary[]; filtersApplied: boolean; hasMore: boolean; nextCursor?: string }>(
`/api/human-item-summaries${queryString(filters)}`,
options,
),
poolCounts: (
filters: HumanItemFilters & { reviewedStageIds?: string[] } = {},
options: { signal?: AbortSignal } = {},
) =>
apiRequest<{ counts: Record<"available" | "mine" | "reviewed" | "all", number>; filtersApplied: boolean }>(
`/api/human-item-counts${queryString(filters)}`,
options,
),
listAllItems,
listAcceptancePools: async (
filters: Omit<HumanItemFilters, "stageId" | "pool">,
@ -110,11 +129,11 @@ export const humanLoopApi = {
);
return { available, mine, all: [], reviewed };
},
getItem: (id: string) =>
getItem: (id: string, options: { signal?: AbortSignal } = {}) =>
apiRequest<{
item: HumanItem;
acceptanceReviews?: { taskId: string; revision: number; decision: string; comment?: string; createdAt: string }[];
}>(`/api/human-items/${encodeURIComponent(id)}`),
}>(`/api/human-items/${encodeURIComponent(id)}`, options),
subjectContentURL: (id: string, side: "left" | "right" = "left") =>
`/api/human-items/${encodeURIComponent(id)}/subject-content${side === "right" ? "?side=right" : ""}`,
evidenceContentURL: (id: string, artifactId: string) =>

View File

@ -2266,6 +2266,15 @@ export interface HumanItem {
display?: HumanTaskDisplayContext;
}
/** Explicit list-only identity. A workbench must load HumanItem detail first. */
export type HumanPoolSummary = Omit<HumanItem, "subject"> & {
kind: "pool-summary";
durationSeconds?: number;
subject: Omit<HumanItem["subject"], "payload">;
};
export const isHumanPoolSummary = (item: HumanItem | HumanPoolSummary): item is HumanPoolSummary =>
"kind" in item && item.kind === "pool-summary";
export interface HumanReject {
id: string;
organizationId: string;

View File

@ -39,7 +39,7 @@ try {
return route.fulfill({ status: 409, json: { error: "任务已被其他人领取。" } });
}
if (!["GET", "HEAD"].includes(r.method())) report.writes.push(u.pathname);
if (u.pathname === "/api/human-items") {
if (["/api/human-items", "/api/human-item-summaries"].includes(u.pathname)) {
report.requests.push(Object.fromEntries(u.searchParams));
if (u.searchParams.get("pool") === "available" && u.searchParams.get("search") === "FYLX_SEARCH_0612")
return route.fulfill({ status: 504, json: { error: "合成可领取池超时" } });
@ -63,8 +63,13 @@ try {
cards = page.getByTestId("annotation-task-card"),
more = page.getByTestId("task-pool-show-more");
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 200);
assert.match(await available.innerText(), /已加载 200\+/);
report.cases.push("bounded initial200 and explicitly loaded count");
await page.waitForFunction(() =>
document.querySelector('[data-testid="human-task-pool-available"]').textContent.includes("612"),
);
assert.match(await available.innerText(), /612/);
assert.match(await page.getByTestId("human-task-pool-mine").innerText(), /1/);
assert.match(await page.getByTestId("human-task-pool-reviewed").innerText(), /1/);
report.cases.push("bounded initial200 with independent totals for unvisited tabs");
await more.click();
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 400);
report.cases.push("available cursor loads next200");

View File

@ -90,7 +90,10 @@ try {
continuationWrites.push(route.request().postDataJSON()?.action);
if (u.pathname.endsWith("/review-session") && route.request().method() === "POST")
sessionWrites.push(route.request().postDataJSON()?.action);
if (blockPoolReads && u.pathname === "/api/human-items") {
if (
blockPoolReads &&
["/api/human-items", "/api/human-item-summaries", "/api/human-item-counts"].includes(u.pathname)
) {
unexpectedPoolReads.push(u.search);
return route.fulfill({ status: 503, json: { error: "Synthetic pool outage during layout change" } });
}

View File

@ -348,3 +348,28 @@ test("my-tasks count marks itself partial while another sidebar page exists", as
assert.match(tabs, /blindAnnotator && value === "mine" && minePoolCursorRef\.current/);
assert.match(tabs, /\? "\+"\s*:\s*""/);
});
test("summary protocol preserves identity and falls back only on missing routes", async () => {
const { loadHumanPoolList } = await import("../../src/lib/annotation-project-loading.js");
let legacy = 0;
const item = { kind: "pool-summary", id: "i", version: 7, canClaim: false, subject: { itemKey: "part" } };
const human = {
listSummariesPage: async () => ({ summaries: [item], filtersApplied: true }),
listItemsPage: async () => {
legacy++;
return { items: [] };
},
};
assert.equal((await loadHumanPoolList(human, {}, {})).items[0], item);
assert.equal(legacy, 0);
human.listSummariesPage = async () => {
throw Object.assign(new Error("missing"), { status: 404 });
};
await loadHumanPoolList(human, {}, {});
assert.equal(legacy, 1);
human.listSummariesPage = async () => {
throw Object.assign(new Error("denied"), { status: 403 });
};
await assert.rejects(loadHumanPoolList(human, {}, {}), /denied/);
assert.equal(legacy, 1);
});

View File

@ -0,0 +1,22 @@
import test from "node:test";
import assert from "node:assert/strict";
import { reviewItemForSelection } from "../../src/lib/review-item-selection.js";
test("aggregate summaries wait for matching full detail before editing", () => {
const summary = { kind: "pool-summary", id: "aggregate", version: 3 };
const unrelated = { id: "annotation", version: 9, subject: { payload: { draft: "preserved" } } };
assert.equal(reviewItemForSelection(summary, undefined, unrelated), undefined);
assert.equal(reviewItemForSelection(summary, unrelated, unrelated), undefined);
assert.equal(reviewItemForSelection(summary, { id: "aggregate", version: 2 }, unrelated), undefined);
assert.equal(reviewItemForSelection(summary, summary, unrelated), undefined);
const detail = { id: "aggregate", version: 3, subject: { payload: { items: [{ id: "source" }] } } };
assert.equal(reviewItemForSelection(summary, detail, unrelated), detail);
assert.equal(detail.subject.payload.items[0].id, "source");
});
test("legacy package and annotation selections keep their existing detail path", () => {
const legacy = { id: "legacy-package-task", version: 2 };
assert.equal(reviewItemForSelection(undefined, undefined, legacy), legacy);
const full = { id: "aggregate", version: 4, subject: { payload: { items: [] } } };
assert.equal(reviewItemForSelection(full, undefined, legacy), full);
});