forked from longtao.wu/droboticflow
⚡ Optimize task pool reads and scoped reporting
This commit is contained in:
parent
8c2ae878a2
commit
88526258c9
@ -6,3 +6,20 @@ Base: origin/dev @ 5375074a, fast-forwarded to deployed 4170c74b before changes
|
||||
Scope: Query summaries/search planning, explicit list summaries and scoped counts, reliable derived-data maintenance/backfill, incremental reporting/snapshot reads, regression/performance evidence.
|
||||
Out of scope: Production data/index writes, push, PR and deployment until separately authorized; unrelated local work; permission relaxation, truncated history or changed search semantics.
|
||||
Risk: Derived-state freshness and recovery, mixed clients, old payload search, authoritative claim permission, ordering/cursors, dates and supplier/source deduplication. Query speed must meet representative isolated concurrent p95 <= 2 seconds before release preparation; functional checks alone are insufficient.
|
||||
|
||||
## Implementation and compatibility
|
||||
|
||||
- Source-owned `poolSearch` fields are stamped atomically with HumanItem/Task/Project/Run/Capture writes after supplier scrubbing. Search narrows candidates before the existing joins/sorts; the final matcher and server eligibility remain authoritative. Missing/stale metadata uses the existing pipeline. Broad reference sets never truncate results.
|
||||
- Explicit `/api/human-item-summaries` and `/api/human-item-counts` protocols; legacy item API remains intact. Acceptance detail lives separately from list summaries. Counts traverse all authorized pages and do not return partial totals after errors.
|
||||
- Summary backfill is an explicit, read-only-by-default CLI. Full-document CAS, idempotent reruns, counters only, 64 KiB per-document summary ceiling; no startup/GET writes, index changes or customer-body rewrites. Production apply still needs authorization. Old servers invalidate summaries through source guards; state-only review updates preserve valid summaries without rewriting annotations.
|
||||
- Statistics reuse the existing shared immutable fact snapshots and add bounded (4 MiB / 128 entries) generation/query-scoped compressed result reuse and concurrent request coalescing. Personal/supplier scope, dates, timezone identity and generation are in the key; decoding/calculation remain bounded. This does NOT implement event-driven source ingestion: existing 120-second projected-source refresh/recovery stays in place. Its 5,000-part/25,000-event cold build measured 240 ms locally; there is no measured reason yet to replace that recovery path.
|
||||
|
||||
## Evidence so far (not release acceptance)
|
||||
|
||||
- Production base 4170c74b, fresh source preserved. Main and unrelated worktrees untouched.
|
||||
- Mongo modern/legacy search parity, payload-only legacy names, tenant poisoning, literal regex characters, category/cursor cases, dry run/retry, old-writer invalidation/rebuild, project rename and state-only review metadata passed locally.
|
||||
- API summary/full metadata equivalence, shared acceptance revocation and >1,000-row counts passed. Browser search/pagination/retry/unvisited-tab totals passed; quality/acceptance/legacy receipt focus and unsaved-opinion flows passed.
|
||||
- Dense isolated comparison, identical 81 actors, source payloads, 10-second warmup/60-second measurement: base had 52 errors; candidate 1,765 requests/zero errors. Search p95 7.4–12.4 s -> 257–358 ms; common successful identity sets matched for 1,691 samples, with no differences. Some base actor/endpoint pairs had no successful reference and remain covered by separate equivalence tests, not by that comparison.
|
||||
- Mongo reply bytes 7.21 GB -> 0.119 GB in that synthetic mix; do not extrapolate it into production savings. Compression settings fixed; no media. CPU/heap readings include the in-process harness, not standalone Pod capacity.
|
||||
- Statistics isolated 5,000 parts/25,000 events: cold build 240 ms, warm p95 30 ms, 121-request/4-handler cohort burst p95 60 ms, zero history reloads during warm burst.
|
||||
- First task gate caught legacy no-hit foreign-ID transfer exceeding 64 KiB. Fixed with a tiny legacy-readiness probe; legacy/modern/large-payload Mongo and snapshot regressions passed afterwards. Final task/integration and prolonged repeated performance checks remain pending. Keep this branch local; no production changes, push, PR or deployment authorized for this task.
|
||||
|
||||
46
cmd/pool-search-backfill/main.go
Normal file
46
cmd/pool-search-backfill/main.go
Normal file
@ -0,0 +1,46 @@
|
||||
// Explicit derived-field maintenance. Production execution requires the
|
||||
// project's separate customer-data write authorization; default is read-only.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
database := flag.String("database", "", "database name")
|
||||
org := flag.String("organization", "", "one organization ID")
|
||||
uriEnv := flag.String("uri-env", "POOL_BACKFILL_MONGO_URI", "environment variable containing Mongo URI")
|
||||
apply := flag.Bool("apply", false, "write derived search fields; requires prior authorization for production")
|
||||
timeout := flag.Duration("timeout", 30*time.Minute, "total operation deadline")
|
||||
flag.Parse()
|
||||
if *database == "" || *org == "" || os.Getenv(*uriEnv) == "" || *timeout <= 0 {
|
||||
fmt.Fprintln(os.Stderr, "database, organization and URI environment variable are required")
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv(*uriEnv)))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "Mongo initialization failed")
|
||||
os.Exit(1)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
result, err := store.NewMongoStore(client.Database(*database)).BackfillPoolSearchMetadata(ctx, *org, *apply)
|
||||
// Diagnostics contain counters only, never records or credential-bearing errors.
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"apply": *apply, "counts": result, "complete": err == nil && result.Conflicts == 0 && result.Unsupported == 0 && (*apply || result.Scanned == 0)})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "backfill failed; rerun after resolving the connection or timeout")
|
||||
os.Exit(1)
|
||||
}
|
||||
if result.Conflicts > 0 || result.Unsupported > 0 {
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
@ -62,7 +62,7 @@ func outputDateRange(r *http.Request, now time.Time) (time.Time, time.Time, *tim
|
||||
if zone == "" {
|
||||
zone = "Asia/Shanghai"
|
||||
}
|
||||
loc, e := time.LoadLocation(zone)
|
||||
loc, e := workforcestats.ReportingLocation(zone)
|
||||
if e != nil {
|
||||
return time.Time{}, time.Time{}, nil, fmt.Errorf("invalid time zone")
|
||||
}
|
||||
|
||||
@ -224,7 +224,11 @@ func (resolver *humanDisplayResolver) enrichItem(ctx context.Context, item *doma
|
||||
}
|
||||
if len(values) == 0 {
|
||||
appendCaptureDirectory(&values, domain.CaptureDirectoryFromArtifactValue(item.Subject.ItemKey))
|
||||
humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values)
|
||||
if item.PoolSearchCurrent() {
|
||||
values = append(values, item.PoolSearch.PayloadDirectories...)
|
||||
} else {
|
||||
humanCaptureDirectoriesFromPayload(item.Subject.Payload, &values)
|
||||
}
|
||||
}
|
||||
// Task artifacts are a legacy fallback, not another source to merge with
|
||||
// an already resolved capture. Avoid fetching the full annotation document
|
||||
|
||||
@ -491,7 +491,7 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
if nextCursor != nil && !exhaustedAll && !hasMore && len(filtered) < limit {
|
||||
hasMore = true
|
||||
}
|
||||
if pool == "reviewed" {
|
||||
if pool == "reviewed" && poolReadMode(request) != humanPoolReadCount {
|
||||
ids := make([]string, 0, len(filtered))
|
||||
for _, item := range filtered {
|
||||
ids = append(ids, item.ID)
|
||||
@ -517,6 +517,13 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
}
|
||||
}
|
||||
}
|
||||
if poolReadMode(request) == humanPoolReadCount {
|
||||
page := &humanItemListPage{Items: filtered, HasMore: hasMore}
|
||||
if hasMore {
|
||||
page.NextCursor = encodeHumanItemCursor(nextCursor)
|
||||
}
|
||||
return page, request
|
||||
}
|
||||
resolver := newHumanDisplayResolver(server)
|
||||
if err := resolver.primeRuns(request.Context(), filtered); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
@ -528,7 +535,12 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
return nil, request
|
||||
}
|
||||
server.anonymizeV58AcceptanceItem(request.Context(), &filtered[index])
|
||||
if err := server.projectRejectedAcceptanceData(request.Context(), &filtered[index]); err != nil {
|
||||
if err := func() error {
|
||||
if poolReadMode(request) == humanPoolReadSummary {
|
||||
return nil
|
||||
}
|
||||
return server.projectRejectedAcceptanceData(request.Context(), &filtered[index])
|
||||
}(); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return nil, request
|
||||
}
|
||||
@ -618,6 +630,9 @@ func (server *server) filterHumanPoolItem(request *http.Request, item domain.Hum
|
||||
// changes between reads sends this page through the original single-read path;
|
||||
// stale metadata must not govern a newer assignment, stage or permission scope.
|
||||
func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain.HumanItemRecord, pool string, query domain.HumanItemQuery, remaining int, batches map[string]*domain.HumanBatchRecord) ([]domain.HumanItemRecord, error) {
|
||||
if poolReadMode(request) == humanPoolReadCount {
|
||||
return items, nil
|
||||
}
|
||||
ids := []string{}
|
||||
selected := map[string]int{}
|
||||
for index, item := range items {
|
||||
@ -626,6 +641,9 @@ func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain
|
||||
return nil, err
|
||||
}
|
||||
if visible {
|
||||
if poolReadMode(request) == humanPoolReadSummary && item.PoolSearchCurrent() {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, item.ID)
|
||||
selected[item.ID] = index
|
||||
if len(ids) == remaining {
|
||||
|
||||
207
internal/api/human_pool_reads.go
Normal file
207
internal/api/human_pool_reads.go
Normal file
@ -0,0 +1,207 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type humanPoolReadMode int
|
||||
|
||||
const (
|
||||
humanPoolReadFull humanPoolReadMode = iota
|
||||
humanPoolReadSummary
|
||||
humanPoolReadCount
|
||||
)
|
||||
|
||||
type humanPoolReadModeKey struct{}
|
||||
|
||||
func poolReadMode(r *http.Request) humanPoolReadMode {
|
||||
mode, _ := r.Context().Value(humanPoolReadModeKey{}).(humanPoolReadMode)
|
||||
return mode
|
||||
}
|
||||
|
||||
// Summary is an explicit opt-in protocol. Legacy clients retain their item
|
||||
// response; workbenches must fetch detail before opening a summary.
|
||||
type humanPoolSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
DurationSeconds float64 `json:"durationSeconds,omitempty"`
|
||||
SourceValidationStatus string `json:"sourceValidationStatus,omitempty"`
|
||||
TraceSummary *domain.HumanTraceSummary `json:"traceSummary,omitempty"`
|
||||
ReviewHistory *domain.HumanReviewHistory `json:"reviewHistory,omitempty"`
|
||||
CanClaim *bool `json:"canClaim,omitempty"`
|
||||
AcceptanceClaim *domain.AcceptanceClaimAccess `json:"acceptanceClaim,omitempty"`
|
||||
ReviewHistoryStatus string `json:"reviewHistoryStatus,omitempty"`
|
||||
ReviewOperationID string `json:"reviewOperationId,omitempty"`
|
||||
ProcessingScope string `json:"processingScope,omitempty"`
|
||||
ProcessingProfile string `json:"processingProfile,omitempty"`
|
||||
ID string `json:"id"`
|
||||
OrganizationID string `json:"organizationId"`
|
||||
PluginID string `json:"pluginId"`
|
||||
BatchID string `json:"batchId"`
|
||||
ProjectID string `json:"projectId"`
|
||||
SupplierID string `json:"supplierId,omitempty"`
|
||||
WorkflowID string `json:"workflowId"`
|
||||
RunID string `json:"runId"`
|
||||
GateNodeID string `json:"gateNodeId"`
|
||||
LegacyTaskID string `json:"legacyTaskId,omitempty"`
|
||||
LegacyAuditTaskID string `json:"legacyAuditTaskId,omitempty"`
|
||||
Subject domain.HumanSubjectRef `json:"subject"`
|
||||
CurrentStageID string `json:"currentStageId"`
|
||||
Status domain.HumanItemStatus `json:"status"`
|
||||
AssignedUserID string `json:"assignedUserId,omitempty"`
|
||||
LegacyAssignment map[string]string `json:"legacyAssignment,omitempty"`
|
||||
CurrentRevision int `json:"currentRevision"`
|
||||
ReviewRound int `json:"reviewRound"`
|
||||
MaxReworkRounds int `json:"maxReworkRounds"`
|
||||
Version int `json:"version"`
|
||||
LatestAuthorUserID string `json:"latestAuthorUserId,omitempty"`
|
||||
PendingCombinedSubmissionID string `json:"pendingCombinedSubmissionId,omitempty"`
|
||||
PendingCombinedPayloadSHA string `json:"pendingCombinedPayloadSha,omitempty"`
|
||||
ReleasePartitionID string `json:"releasePartitionId,omitempty"`
|
||||
ClaimedAt string `json:"claimedAt,omitempty"`
|
||||
SubmittedAt string `json:"submittedAt,omitempty"`
|
||||
AcceptanceSampleProbability *float64 `json:"acceptanceSampleProbability,omitempty"`
|
||||
AcceptanceSampleValue *float64 `json:"acceptanceSampleValue,omitempty"`
|
||||
AcceptancePolicyRevision int `json:"acceptancePolicyRevision,omitempty"`
|
||||
AcceptedAt string `json:"acceptedAt,omitempty"`
|
||||
AcceptedByUserID string `json:"acceptedByUserId,omitempty"`
|
||||
RejectedAt string `json:"rejectedAt,omitempty"`
|
||||
CancelledAt string `json:"cancelledAt,omitempty"`
|
||||
CancelledByUserID string `json:"cancelledByUserId,omitempty"`
|
||||
CancellationReason string `json:"cancellationReason,omitempty"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt"`
|
||||
Display *domain.HumanTaskDisplayContext `json:"display,omitempty"`
|
||||
}
|
||||
|
||||
func summarizeHumanPoolItem(i domain.HumanItemRecord) humanPoolSummary {
|
||||
seconds := float64(0)
|
||||
if i.PoolSearchCurrent() {
|
||||
seconds = i.PoolSearch.DurationSeconds
|
||||
} else {
|
||||
for _, value := range listValue(i.Subject.Payload["items"]) {
|
||||
seconds += max(0, floatValue(mapValue(mapValue(mapValue(value)["materialization"])["coverage"])["durationSeconds"]))
|
||||
}
|
||||
}
|
||||
subject := i.Subject
|
||||
subject.Payload = nil
|
||||
return humanPoolSummary{Kind: "pool-summary", DurationSeconds: seconds,
|
||||
SourceValidationStatus: i.SourceValidationStatus,
|
||||
TraceSummary: i.TraceSummary,
|
||||
ReviewHistory: i.ReviewHistory,
|
||||
CanClaim: i.CanClaim,
|
||||
AcceptanceClaim: i.AcceptanceClaim,
|
||||
ReviewHistoryStatus: i.ReviewHistoryStatus,
|
||||
ReviewOperationID: i.ReviewOperationID,
|
||||
ProcessingScope: i.ProcessingScope,
|
||||
ProcessingProfile: i.ProcessingProfile,
|
||||
ID: i.ID,
|
||||
OrganizationID: i.OrganizationID,
|
||||
PluginID: i.PluginID,
|
||||
BatchID: i.BatchID,
|
||||
ProjectID: i.ProjectID,
|
||||
SupplierID: i.SupplierID,
|
||||
WorkflowID: i.WorkflowID,
|
||||
RunID: i.RunID,
|
||||
GateNodeID: i.GateNodeID,
|
||||
LegacyTaskID: i.LegacyTaskID,
|
||||
LegacyAuditTaskID: i.LegacyAuditTaskID,
|
||||
Subject: subject,
|
||||
CurrentStageID: i.CurrentStageID,
|
||||
Status: i.Status,
|
||||
AssignedUserID: i.AssignedUserID,
|
||||
LegacyAssignment: i.LegacyAssignment,
|
||||
CurrentRevision: i.CurrentRevision,
|
||||
ReviewRound: i.ReviewRound,
|
||||
MaxReworkRounds: i.MaxReworkRounds,
|
||||
Version: i.Version,
|
||||
LatestAuthorUserID: i.LatestAuthorUserID,
|
||||
PendingCombinedSubmissionID: i.PendingCombinedSubmissionID,
|
||||
PendingCombinedPayloadSHA: i.PendingCombinedPayloadSHA,
|
||||
ReleasePartitionID: i.ReleasePartitionID,
|
||||
ClaimedAt: i.ClaimedAt,
|
||||
SubmittedAt: i.SubmittedAt,
|
||||
AcceptanceSampleProbability: i.AcceptanceSampleProbability,
|
||||
AcceptanceSampleValue: i.AcceptanceSampleValue,
|
||||
AcceptancePolicyRevision: i.AcceptancePolicyRevision,
|
||||
AcceptedAt: i.AcceptedAt,
|
||||
AcceptedByUserID: i.AcceptedByUserID,
|
||||
RejectedAt: i.RejectedAt,
|
||||
CancelledAt: i.CancelledAt,
|
||||
CancelledByUserID: i.CancelledByUserID,
|
||||
CancellationReason: i.CancellationReason,
|
||||
CreatedAt: i.CreatedAt,
|
||||
UpdatedAt: i.UpdatedAt,
|
||||
Display: i.Display,
|
||||
}
|
||||
}
|
||||
func (s *server) handleHumanPoolSummaries(w http.ResponseWriter, r *http.Request) {
|
||||
r = r.WithContext(context.WithValue(r.Context(), humanPoolReadModeKey{}, humanPoolReadSummary))
|
||||
page, _ := s.readHumanItemsList(w, r)
|
||||
if page == nil {
|
||||
return
|
||||
}
|
||||
summaries := make([]humanPoolSummary, 0, len(page.Items))
|
||||
for _, item := range page.Items {
|
||||
summaries = append(summaries, summarizeHumanPoolItem(item))
|
||||
}
|
||||
renderJSON(w, 200, map[string]any{"summaries": summaries, "filtersApplied": true, "hasMore": page.HasMore, "nextCursor": page.NextCursor})
|
||||
}
|
||||
|
||||
// Count every authorized page, without payload hydration or display joins.
|
||||
// Errors/cancellation return an error, never a partial total. No cross-request
|
||||
// cache: revocation and self-review rules are checked by the same list filter.
|
||||
func (s *server) handleHumanPoolCounts(w http.ResponseWriter, r *http.Request) {
|
||||
counts := map[string]int{}
|
||||
pools := []string{"available", "mine"}
|
||||
principal, _ := currentPrincipal(r)
|
||||
if annotationManager(principal) || principal != nil && (principal.User.Role == domain.UserRoleQualityReviewer || principal.User.Role == domain.UserRoleAcceptanceReviewer) {
|
||||
pools = append(pools, "reviewed")
|
||||
}
|
||||
if annotationManager(principal) {
|
||||
pools = append(pools, "all")
|
||||
}
|
||||
for _, pool := range pools {
|
||||
count := 0
|
||||
cursor := ""
|
||||
for {
|
||||
if r.Context().Err() != nil {
|
||||
renderError(w, 500, r.Context().Err())
|
||||
return
|
||||
}
|
||||
next := r.Clone(context.WithValue(r.Context(), humanPoolReadModeKey{}, humanPoolReadCount))
|
||||
u := *r.URL
|
||||
next.URL = &u
|
||||
query := next.URL.Query()
|
||||
if pool == "reviewed" {
|
||||
query.Del("stageId")
|
||||
query.Del("stageIds")
|
||||
if stages := query.Get("reviewedStageIds"); stages != "" {
|
||||
query.Set("stageIds", stages)
|
||||
}
|
||||
}
|
||||
query.Del("reviewedStageIds")
|
||||
query.Set("pool", pool)
|
||||
query.Set("limit", "1000")
|
||||
query.Set("cursor", cursor)
|
||||
next.URL.RawQuery = query.Encode()
|
||||
page, readRequest := s.readHumanItemsList(w, next)
|
||||
if page == nil {
|
||||
return
|
||||
}
|
||||
r = r.WithContext(readRequest.Context())
|
||||
count += len(page.Items)
|
||||
if !page.HasMore {
|
||||
break
|
||||
}
|
||||
if page.NextCursor == "" || page.NextCursor == cursor {
|
||||
renderJSON(w, 500, map[string]string{"error": "任务数量暂时无法核对,请重试。"})
|
||||
return
|
||||
}
|
||||
cursor = page.NextCursor
|
||||
}
|
||||
counts[pool] = count
|
||||
}
|
||||
renderJSON(w, 200, map[string]any{"counts": counts, "filtersApplied": true})
|
||||
}
|
||||
101
internal/api/human_pool_reads_test.go
Normal file
101
internal/api/human_pool_reads_test.go
Normal file
@ -0,0 +1,101 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHumanPoolSummaryAndCountPreserveAuthorization(t *testing.T) {
|
||||
f := sharedAcceptanceFixture(t)
|
||||
f.item.Subject.Payload["hugeDetail"] = strings.Repeat("private draft ", 100000)
|
||||
if err := f.store.UpsertHumanItem(f.ctx, f.item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
full := f.request("GET", "/api/human-items?pluginId=annotation-qa&pool=available", f.reviewer, nil)
|
||||
summary := f.request("GET", "/api/human-item-summaries?pluginId=annotation-qa&pool=available", f.reviewer, nil)
|
||||
if full.Code != 200 || summary.Code != 200 {
|
||||
t.Fatalf("full=%d summary=%d %s", full.Code, summary.Code, summary.Body.String())
|
||||
}
|
||||
var before struct {
|
||||
Items []domain.HumanItemRecord `json:"items"`
|
||||
}
|
||||
var after struct {
|
||||
Summaries []humanPoolSummary `json:"summaries"`
|
||||
}
|
||||
if err := json.Unmarshal(full.Body.Bytes(), &before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(summary.Body.Bytes(), &after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Compare the public metadata contract independently of the DTO converter.
|
||||
var fullJSON, summaryJSON map[string]any
|
||||
_ = json.Unmarshal(full.Body.Bytes(), &fullJSON)
|
||||
_ = json.Unmarshal(summary.Body.Bytes(), &summaryJSON)
|
||||
for index, value := range fullJSON["items"].([]any) {
|
||||
expected := value.(map[string]any)
|
||||
delete(expected["subject"].(map[string]any), "payload")
|
||||
actual := summaryJSON["summaries"].([]any)[index].(map[string]any)
|
||||
delete(actual, "kind")
|
||||
delete(actual, "durationSeconds")
|
||||
if !reflect.DeepEqual(expected, actual) {
|
||||
t.Fatalf("public list metadata changed: expected=%v actual=%v", expected, actual)
|
||||
}
|
||||
}
|
||||
if len(before.Items) != len(after.Summaries) || len(after.Summaries) == 0 {
|
||||
t.Fatal("summary lost rows")
|
||||
}
|
||||
for i, item := range before.Items {
|
||||
want := summarizeHumanPoolItem(item)
|
||||
if !reflect.DeepEqual(want, after.Summaries[i]) {
|
||||
t.Fatalf("summary metadata changed: %#v %#v", want, after.Summaries[i])
|
||||
}
|
||||
}
|
||||
if len(summary.Body.Bytes()) >= 10000 || strings.Contains(summary.Body.String(), "private draft") {
|
||||
t.Fatal("summary returned detail")
|
||||
}
|
||||
counts := f.request("GET", "/api/human-item-counts?pluginId=annotation-qa&stageIds=acceptance,prepackage_acceptance&reviewedStageIds=acceptance,prepackage_acceptance", f.reviewer, nil)
|
||||
var totals struct {
|
||||
Counts map[string]int `json:"counts"`
|
||||
}
|
||||
if counts.Code != 200 || json.Unmarshal(counts.Body.Bytes(), &totals) != nil || totals.Counts["available"] != len(before.Items) {
|
||||
t.Fatalf("counts=%d %s", counts.Code, counts.Body.String())
|
||||
}
|
||||
if _, err := f.store.SetSupplierAcceptanceAuthorization(f.ctx, f.item.OrganizationID, f.reviewer.SupplierID, false, 1, f.admin.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, path := range []string{"/api/human-item-summaries?pool=available&pluginId=annotation-qa", "/api/human-item-counts?pluginId=annotation-qa"} {
|
||||
result := f.request("GET", path, f.reviewer, nil)
|
||||
if result.Code != 200 || strings.Contains(result.Body.String(), `"canClaim":true`) {
|
||||
t.Fatalf("revoked result %d %s", result.Code, result.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanPoolCountsTraversesPagesWithoutHydratingPayloads(t *testing.T) {
|
||||
f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1)
|
||||
for n := 0; n < 1005; n++ {
|
||||
item := f.item
|
||||
item.ID = fmt.Sprintf("count-%04d", n)
|
||||
item.CreatedAt = time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339Nano)
|
||||
item.Subject.Payload = map[string]any{"hugeDetail": strings.Repeat("x", 10000)}
|
||||
if err := f.store.UpsertHumanItem(f.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result := f.request("GET", "/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review", f.quality, nil)
|
||||
var body struct {
|
||||
Counts map[string]int `json:"counts"`
|
||||
}
|
||||
if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &body) != nil || body.Counts["available"] != 1006 {
|
||||
t.Fatalf("count=%d %s", result.Code, result.Body.String())
|
||||
}
|
||||
if result.Body.Len() > 500 {
|
||||
t.Fatal("count hydrated details")
|
||||
}
|
||||
}
|
||||
@ -84,7 +84,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
if totalActors > 250 {
|
||||
t.Fatal("local read mix supports at most 250 actors")
|
||||
}
|
||||
availablePerStage := seconds("POLL_POOL_AVAILABLE_PER_STAGE", 8)
|
||||
if availablePerStage > 250 {
|
||||
t.Fatal("available fixtures limited to 250 per stage and supplier")
|
||||
}
|
||||
claimCounts := os.Getenv("POLL_POOL_CLAIM_COUNTS") == "1"
|
||||
readModels := os.Getenv("POLL_POOL_READMODELS") == "1"
|
||||
|
||||
mode := os.Getenv("POLL_POOL_MODE")
|
||||
if mode == "" {
|
||||
@ -214,6 +219,14 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
t.Fatal("run payload must be 0..2048 KiB")
|
||||
}
|
||||
}
|
||||
taskPayloadKiB := 0
|
||||
if raw := os.Getenv("POLL_POOL_TASK_PAYLOAD_KIB"); raw != "" {
|
||||
taskPayloadKiB, err = strconv.Atoi(raw)
|
||||
must(err)
|
||||
if taskPayloadKiB < 0 || taskPayloadKiB > 2048 {
|
||||
t.Fatal("task payload must be 0..2048 KiB")
|
||||
}
|
||||
}
|
||||
seededRuns := map[string]bool{}
|
||||
seed := func(id, p, s, stage, owner string, status domain.HumanItemStatus) {
|
||||
batchID := "batch-" + p + s
|
||||
@ -237,7 +250,12 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
taskID := "task-" + id
|
||||
must(ds.UpsertAnnotationTask(ctx, domain.AnnotationTaskRecord{ID: taskID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, SupplierID: s, TemplateVersion: 6, Status: domain.AnnotationTaskStatusPendingAnnotation, SourceVideo: "synthetic/camera_left.mp4"}))
|
||||
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}}}))
|
||||
itemPayload := map[string]any{"captureDirectory": "synthetic-" + id, "sliceMergeEvidence": payload, "sliceWorkflow": map[string]any{"status": "completed", "phase": "annotate"}}
|
||||
if taskPayloadKiB > 0 {
|
||||
content := strings.Repeat(strings.Join(payload, ""), taskPayloadKiB/32+1)[:taskPayloadKiB*1024]
|
||||
itemPayload["items"] = []map[string]any{{"annotation": map[string]any{"syntheticPayload": content}, "materialization": map[string]any{"coverage": map[string]any{"durationSeconds": 900}}}}
|
||||
}
|
||||
must(ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: id, RunID: runID, OrganizationID: org, ProjectID: p, WorkflowID: "workflow-" + p, PluginID: "annotation-qa", SupplierID: s, BatchID: batchID, LegacyTaskID: taskID, AssignedUserID: owner, LatestAuthorUserID: "synthetic-author", CurrentStageID: stage, Status: status, Version: 1, CreatedAt: "2026-10-01T00:00:00Z", Subject: domain.HumanSubjectRef{ItemKey: "synthetic/" + id + "/camera_left.mp4", Payload: itemPayload}}))
|
||||
}
|
||||
for _, a := range actors {
|
||||
if a.Role == "admin" {
|
||||
@ -269,7 +287,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
stages = append(stages, "annotation_combined")
|
||||
}
|
||||
for _, stage := range stages {
|
||||
for n := 0; n < 8; n++ {
|
||||
for n := 0; n < availablePerStage; n++ {
|
||||
seed(fmt.Sprintf("available-%s-%s-%s-%d", p, s, stage, n), p, s, stage, "", domain.HumanItemReady)
|
||||
}
|
||||
}
|
||||
@ -335,6 +353,13 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
if authorizationState == "initialized" {
|
||||
must(ds.InitializeSupplierProjectAuthorizations(ctx, org))
|
||||
}
|
||||
if readModels {
|
||||
result, err := ds.BackfillPoolSearchMetadata(ctx, org, true)
|
||||
must(err)
|
||||
if result.Unsupported > 0 || result.Conflicts > 0 {
|
||||
t.Fatalf("summary preparation incomplete: %+v", result)
|
||||
}
|
||||
}
|
||||
handler := NewHandler(ds, nil)
|
||||
var mu sync.Mutex
|
||||
samples := []pollLoadSample{}
|
||||
@ -438,7 +463,11 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
result := map[string]string{}
|
||||
for _, pool := range []string{"available", "mine", "reviewed"} {
|
||||
base := "/api/human-items?pluginId=annotation-qa&pool=" + pool + scope
|
||||
endpoint := "/api/human-items"
|
||||
if readModels {
|
||||
endpoint = "/api/human-item-summaries"
|
||||
}
|
||||
base := endpoint + "?pluginId=annotation-qa&pool=" + pool + scope
|
||||
if a.Role == "quality" {
|
||||
result["quality."+scopeName+"."+pool] = base + "&stageIds=quality_review%2Csemantic_review"
|
||||
} else {
|
||||
@ -447,6 +476,26 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if readModels && countCycle {
|
||||
stages := "quality_review,semantic_review"
|
||||
reviewed := ""
|
||||
if a.Role == "acceptance" {
|
||||
stages = "acceptance,prepackage_acceptance"
|
||||
reviewed = "&reviewedStageIds=" + stages
|
||||
}
|
||||
result[a.Role+"."+scopeName+".counts"] = "/api/human-item-counts?pluginId=annotation-qa" + scope + "&stageIds=" + stages + reviewed
|
||||
}
|
||||
if os.Getenv("POLL_POOL_SEARCH") == "1" {
|
||||
endpoint := "/api/human-items"
|
||||
if readModels {
|
||||
endpoint = "/api/human-item-summaries"
|
||||
}
|
||||
stage := "quality_review"
|
||||
if a.Role == "acceptance" {
|
||||
stage = "prepackage_acceptance"
|
||||
}
|
||||
result[a.Role+"."+scopeName+".search"] = endpoint + "?pluginId=annotation-qa&pool=available&limit=200" + scope + "&stageId=" + stage + "&search=synthetic-available-" + a.Project + "-" + a.User.SupplierID + "-" + stage + "-0"
|
||||
}
|
||||
return result
|
||||
}
|
||||
// Fail before timing if permissions or fixtures accidentally produce empty work.
|
||||
@ -610,7 +659,7 @@ func TestPollPoolReadMixLocal(t *testing.T) {
|
||||
sort.Float64s(lat)
|
||||
metrics[key] = map[string]any{"requests": len(rows), "p95_ms": lat[(len(lat)*95-1)/100], "max_ms": lat[len(lat)-1], "raw_bytes": raw, "gzip_bytes": gz, "min_items": minRows}
|
||||
}
|
||||
report := map[string]any{"schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
|
||||
report := map[string]any{"read_models": readModels, "search_queries": os.Getenv("POLL_POOL_SEARCH") == "1", "schema": 2, "resource_samples": resourceRows, "process_cpu_seconds": cpuAfter - cpuBefore, "authorization_state": authorizationState, "fixture": "read-mix-v3-released-history", "available_per_stage": availablePerStage, "task_payload_kib": taskPayloadKiB, "run_payload_kib": runPayloadKiB, "author_history_per_reviewer": authorHistory, "mode": mode, "revision": os.Getenv("POLL_POOL_REVISION"), "actors": totalActors, "actor_counts": map[string]int{"annotation": annotators, "quality": quality, "acceptance": acceptance, "admin": 1}, "claim_counts": claimCounts, "warmup_seconds": warmup, "scheduled_seconds": duration, "actual_seconds": actual.Seconds(), "errors": failures, "failure_samples": failureSamples, "metrics": metrics, "mongo_breakdown": observations, "mongo_commands": commands.Load(), "mongo_reply_bytes": mongoBytes.Load(), "mongo_command_ms": float64(mongoNS.Load()) / 1e6, "mongo_write_commands": writes.Load(), "go_heap_before": memBefore.HeapAlloc, "go_heap_after": memAfter.HeapAlloc, "go_heap_after_gc": memAfterGC.HeapAlloc, "go_gc_cycles": memAfter.NumGC - memBefore.NumGC, "go_allocated_bytes": memAfter.TotalAlloc - memBefore.TotalAlloc, "limitations": []string{"handler timing excludes ingress/network/authentication", "gzip computed separately with fixed Go default compression", "no continuation renew, media or workflow writes", "synthetic fixture; not capacity acceptance", "CPU covers handler and in-process load harness, not Mongo/Pod throttling", "resource heap includes retained benchmark request samples; not standalone service RSS"}, "samples": samples}
|
||||
encoded, e := json.MarshalIndent(report, "", " ")
|
||||
must(e)
|
||||
must(os.WriteFile(out, encoded, 0600))
|
||||
|
||||
@ -21,7 +21,7 @@ func productionQualityDates(r *http.Request, now time.Time) (time.Time, time.Tim
|
||||
if zone == "" {
|
||||
zone = "Asia/Shanghai"
|
||||
}
|
||||
loc, e := time.LoadLocation(zone)
|
||||
loc, e := workforcestats.ReportingLocation(zone)
|
||||
if e != nil {
|
||||
return time.Time{}, time.Time{}, nil, e
|
||||
}
|
||||
|
||||
@ -8,6 +8,8 @@ import "github.com/go-chi/chi/v5"
|
||||
func (server *server) mountHumanLoopRoutes(router chi.Router) {
|
||||
router.Get("/api/human-batches", server.handleHumanBatchesList)
|
||||
router.Get("/api/human-items", server.handleHumanItemsList)
|
||||
router.Get("/api/human-item-summaries", server.handleHumanPoolSummaries)
|
||||
router.Get("/api/human-item-counts", server.handleHumanPoolCounts)
|
||||
router.Get("/api/plugins/annotation-qa/claim-projects", server.handleAnnotationClaimProjects)
|
||||
router.Get("/api/plugins/annotation-qa/claim-count", server.handleAnnotationClaimCount)
|
||||
router.Get("/api/human-items/{id}", server.handleHumanItemGet)
|
||||
|
||||
4
internal/api/testdata/routes.golden
vendored
4
internal/api/testdata/routes.golden
vendored
@ -1,7 +1,7 @@
|
||||
# Generated by TestRouteSnapshot; see routes_snapshot_test.go.
|
||||
# access: reach = endpoint reached; otherwise the status the middleware chain returned.
|
||||
global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache]
|
||||
routes=379
|
||||
routes=381
|
||||
DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
|
||||
DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
|
||||
DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
@ -67,6 +67,8 @@ GET /api/external/object-sync/sessions/{sessionId}/items | mw=[(*server).require
|
||||
GET /api/external/object-sync/sessions/{sessionId}/status | mw=[(*server).requireObjectSyncFeature-fm,(*server).objectSyncCredentialAuth-fm,(*server).objectSyncDistributedRateLimit.1,(*server).requireObjectSyncIdempotency-fm] | (*server).handleExternalObjectSyncSessionStatus-fm | anon=401 token=401 super=401 admin=401 supplier=401 annotator=401 quality=401 acceptance=401
|
||||
GET /api/health | mw=[] | (*server).handleHealth-fm | anon=reach token=reach super=reach admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/human-batches | mw=[] | (*server).handleHumanBatchesList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/human-item-counts | mw=[] | (*server).handleHumanPoolCounts-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/human-item-summaries | mw=[] | (*server).handleHumanPoolSummaries-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/human-items | mw=[] | (*server).handleHumanItemsList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/human-items/{id} | mw=[] | (*server).handleHumanItemGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/human-items/{id}/acceptance-misreports | mw=[(*server).requireBusinessRole.1] | (*server).handleAcceptanceMisreports-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=reach acceptance=reach
|
||||
|
||||
@ -148,7 +148,22 @@ type HumanTraceSummary struct {
|
||||
Rejections *int `json:"rejections"`
|
||||
}
|
||||
|
||||
// HumanPoolSearchMetadata contains only source-owned names. It is never public
|
||||
// task content or a permission decision.
|
||||
type HumanPoolSearchMetadata struct {
|
||||
Source map[string]any `bson:"source"`
|
||||
SourceVersion int `bson:"sourceVersion"`
|
||||
SourceUpdatedAt string `bson:"sourceUpdatedAt"`
|
||||
SourceItemKey string `bson:"sourceItemKey"`
|
||||
SourceArtifactID string `bson:"sourceArtifactId"`
|
||||
DurationSeconds float64 `bson:"durationSeconds"`
|
||||
Schema int `bson:"schema"`
|
||||
Text []string `bson:"text"`
|
||||
PayloadDirectories []string `bson:"payloadDirectories"`
|
||||
}
|
||||
|
||||
type HumanItemRecord struct {
|
||||
PoolSearch *HumanPoolSearchMetadata `json:"-" bson:"poolSearch,omitempty"`
|
||||
SourceValidationStatus string `json:"sourceValidationStatus,omitempty" bson:"-"`
|
||||
TraceSummary *HumanTraceSummary `json:"traceSummary,omitempty" bson:"-"`
|
||||
ReviewHistory *HumanReviewHistory `json:"reviewHistory,omitempty" bson:"-"`
|
||||
@ -497,3 +512,10 @@ type HumanRunDisplay struct {
|
||||
TaskID string `json:"taskId" bson:"taskId"`
|
||||
CaptureDirectories []string `json:"captureDirectories" bson:"captureDirectories"`
|
||||
}
|
||||
|
||||
// PoolSearchCurrent also detects an old server writing a record without
|
||||
// refreshing its derived summary during a rolling upgrade or rollback.
|
||||
func (item HumanItemRecord) PoolSearchCurrent() bool {
|
||||
m := item.PoolSearch
|
||||
return m != nil && m.Schema == 1 && m.SourceVersion == item.Version && m.SourceUpdatedAt == item.UpdatedAt && m.SourceItemKey == item.Subject.ItemKey && m.SourceArtifactID == item.Subject.ArtifactID
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"golang.org/x/sync/singleflight"
|
||||
"io"
|
||||
"log"
|
||||
"sort"
|
||||
@ -18,24 +19,27 @@ import (
|
||||
)
|
||||
|
||||
type Reader struct {
|
||||
querySlots chan struct{}
|
||||
backend Backend
|
||||
source Source
|
||||
now func() time.Time
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
attempts map[string]time.Time
|
||||
cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache.
|
||||
cacheBytes int
|
||||
reads chan struct{}
|
||||
queryMemo map[string][]byte
|
||||
queryMemoBytes int
|
||||
queryFlights singleflight.Group
|
||||
querySlots chan struct{}
|
||||
backend Backend
|
||||
source Source
|
||||
now func() time.Time
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
attempts map[string]time.Time
|
||||
cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache.
|
||||
cacheBytes int
|
||||
reads chan struct{}
|
||||
}
|
||||
|
||||
func NewReader(b Backend, source Source) *Reader {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Reader{querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
|
||||
return &Reader{queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
|
||||
}
|
||||
func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() }
|
||||
func scopeKey(org string, projects []string) string {
|
||||
@ -48,7 +52,7 @@ func scopeKey(org string, projects []string) string {
|
||||
|
||||
// Read returns nil while the initial generation is being built. Existing complete
|
||||
// generations remain available during refresh. Pins never silently change data.
|
||||
func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) {
|
||||
func (r *Reader) resolve(ctx context.Context, org string, projects []string, pin string) (*Generation, Info, error) {
|
||||
key, now := scopeKey(org, projects), r.now()
|
||||
state, err := r.backend.OutputSnapshotState(ctx)
|
||||
if err != nil {
|
||||
@ -98,6 +102,22 @@ func (r *Reader) Read(ctx context.Context, org string, projects []string, pin st
|
||||
}
|
||||
return nil, info, nil
|
||||
}
|
||||
info.ID = gen.ID
|
||||
info.AsOf = gen.AsOf
|
||||
info.Stale = now.Sub(gen.AsOf) >= RefreshInterval
|
||||
return gen, info, nil
|
||||
}
|
||||
|
||||
func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) {
|
||||
gen, info, err := r.resolve(ctx, org, projects, pin)
|
||||
if err != nil || gen == nil {
|
||||
return nil, info, err
|
||||
}
|
||||
return r.readResolved(ctx, gen, info, pin)
|
||||
}
|
||||
func (r *Reader) readResolved(ctx context.Context, gen *Generation, info Info, pin string) (*workforcestats.OutputDataset, Info, error) {
|
||||
now := r.now()
|
||||
|
||||
select {
|
||||
case r.reads <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
@ -105,9 +125,17 @@ func (r *Reader) Read(ctx context.Context, org string, projects []string, pin st
|
||||
}
|
||||
defer func() { <-r.reads }()
|
||||
data, err := r.load(ctx, *gen)
|
||||
if err != nil && pin == "" && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge {
|
||||
gen = head.Previous
|
||||
data, err = r.load(ctx, *gen)
|
||||
if err != nil && pin == "" {
|
||||
state, stateErr := r.backend.OutputSnapshotState(ctx)
|
||||
if stateErr == nil {
|
||||
for _, head := range state.Heads {
|
||||
if head.Current != nil && head.Current.ID == gen.ID && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge {
|
||||
gen = head.Previous
|
||||
data, err = r.load(ctx, *gen)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, info, ErrUnavailable
|
||||
@ -297,18 +325,137 @@ func (r *Reader) load(ctx context.Context, g Generation) (*workforcestats.Output
|
||||
|
||||
// Query bounds the entire decode + report calculation, not only Mongo reads.
|
||||
func (r *Reader) Query(ctx context.Context, org string, projects []string, pin string, q workforcestats.OutputQuery) (*workforcestats.OutputReport, Info, error) {
|
||||
gen, info, err := r.resolve(ctx, org, projects, pin)
|
||||
if err != nil || gen == nil {
|
||||
return nil, info, err
|
||||
}
|
||||
// Scope permissions are supplied afresh by the handler. The complete query,
|
||||
// generation and location identity participate in the key, including personal
|
||||
// and supplier isolation. No permissions or mutable datasets are cached.
|
||||
q.Now = gen.AsOf
|
||||
keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)})
|
||||
key := string(keyData)
|
||||
select {
|
||||
case r.querySlots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, Info{}, ctx.Err()
|
||||
return nil, info, ctx.Err()
|
||||
}
|
||||
defer func() { <-r.querySlots }()
|
||||
d, info, err := r.Read(ctx, org, projects, pin)
|
||||
if err != nil || d == nil {
|
||||
return nil, info, err
|
||||
cached := r.memoReport(key)
|
||||
<-r.querySlots
|
||||
if cached != nil {
|
||||
return cached, info, nil
|
||||
}
|
||||
result := workforcestats.QueryOutputDataset(*d, q)
|
||||
return &result, info, nil
|
||||
type answer struct {
|
||||
report *workforcestats.OutputReport
|
||||
info Info
|
||||
}
|
||||
result := r.queryFlights.DoChan(key, func() (any, error) {
|
||||
r.mu.Lock()
|
||||
if r.closed {
|
||||
r.mu.Unlock()
|
||||
return nil, ErrUnavailable
|
||||
}
|
||||
r.wg.Add(1)
|
||||
r.mu.Unlock()
|
||||
defer r.wg.Done()
|
||||
buildCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
select {
|
||||
case r.querySlots <- struct{}{}:
|
||||
case <-buildCtx.Done():
|
||||
return nil, buildCtx.Err()
|
||||
}
|
||||
defer func() { <-r.querySlots }()
|
||||
if report := r.memoReport(key); report != nil {
|
||||
return answer{report, info}, nil
|
||||
}
|
||||
data, actual, err := r.readResolved(buildCtx, gen, info, pin)
|
||||
if err != nil || data == nil {
|
||||
return nil, err
|
||||
}
|
||||
report := workforcestats.QueryOutputDataset(*data, q)
|
||||
// Corrupt-current fallback must not poison the current generation's key.
|
||||
if actual.ID == gen.ID {
|
||||
r.rememberReport(key, &report)
|
||||
}
|
||||
return answer{&report, actual}, nil
|
||||
})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, info, ctx.Err()
|
||||
case value := <-result:
|
||||
if value.Err != nil {
|
||||
return nil, info, value.Err
|
||||
}
|
||||
valueAnswer := value.Val.(answer)
|
||||
select {
|
||||
case r.querySlots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, info, ctx.Err()
|
||||
}
|
||||
defer func() { <-r.querySlots }()
|
||||
// Every caller receives an independent value; UI/report handlers may paginate
|
||||
// or modify the report without mutating another user's response.
|
||||
raw, err := json.Marshal(valueAnswer.report)
|
||||
if err != nil {
|
||||
return nil, info, err
|
||||
}
|
||||
var report workforcestats.OutputReport
|
||||
if err = json.Unmarshal(raw, &report); err != nil {
|
||||
return nil, info, err
|
||||
}
|
||||
return &report, valueAnswer.info, nil
|
||||
}
|
||||
}
|
||||
|
||||
const queryMemoBudget = 4 << 20
|
||||
|
||||
func (r *Reader) memoReport(key string) *workforcestats.OutputReport {
|
||||
r.mu.Lock()
|
||||
payload := r.queryMemo[key]
|
||||
r.mu.Unlock()
|
||||
if payload == nil {
|
||||
return nil
|
||||
}
|
||||
reader, err := gzip.NewReader(bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer reader.Close()
|
||||
var report workforcestats.OutputReport
|
||||
if json.NewDecoder(io.LimitReader(reader, MaxDecodedBytes+1)).Decode(&report) != nil {
|
||||
return nil
|
||||
}
|
||||
return &report
|
||||
}
|
||||
func (r *Reader) rememberReport(key string, report *workforcestats.OutputReport) {
|
||||
raw, err := json.Marshal(report)
|
||||
if err != nil || len(raw) > MaxDecodedBytes {
|
||||
return
|
||||
}
|
||||
var encoded bytes.Buffer
|
||||
writer, _ := gzip.NewWriterLevel(&encoded, gzip.BestSpeed)
|
||||
if _, err = writer.Write(raw); err != nil {
|
||||
return
|
||||
}
|
||||
if writer.Close() != nil || encoded.Len() > queryMemoBudget {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.queryMemo[key] != nil {
|
||||
return
|
||||
}
|
||||
cost := encoded.Len() + len(key) + 128
|
||||
if cost > queryMemoBudget {
|
||||
return
|
||||
}
|
||||
if len(r.queryMemo) >= 128 || r.queryMemoBytes+cost > queryMemoBudget {
|
||||
r.queryMemo = map[string][]byte{}
|
||||
r.queryMemoBytes = 0
|
||||
}
|
||||
r.queryMemo[key] = encoded.Bytes()
|
||||
r.queryMemoBytes += cost
|
||||
}
|
||||
|
||||
func failureCode(err error) string {
|
||||
|
||||
@ -267,3 +267,34 @@ func TestSnapshotScopeCountCannotGrowWithoutBound(t *testing.T) {
|
||||
t.Fatal("scope limit not enforced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportMemoScopeGenerationAndMutationIsolation(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
r := NewReader(&MemoryBackend{}, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
|
||||
return testData(now), nil
|
||||
})
|
||||
defer r.Close()
|
||||
_, generation := ready(t, r, "")
|
||||
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
|
||||
first, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
|
||||
if err != nil || first == nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
first.Totals.Submitted.Count = -100
|
||||
q.Now = now.Add(time.Minute)
|
||||
second, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
|
||||
if err != nil || second.Totals.Submitted.Count != 12 {
|
||||
t.Fatalf("memo mutated: %+v %v", second, err)
|
||||
}
|
||||
q.ScopeUserID = "other"
|
||||
other, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
|
||||
if err != nil || other.Totals.Submitted.Count != 0 {
|
||||
t.Fatalf("scope leaked: %+v %v", other, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
entries, bytes := len(r.queryMemo), r.queryMemoBytes
|
||||
r.mu.Unlock()
|
||||
if entries != 2 || bytes > queryMemoBudget {
|
||||
t.Fatalf("memo entries=%d bytes=%d", entries, bytes)
|
||||
}
|
||||
}
|
||||
|
||||
@ -169,7 +169,11 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
|
||||
}
|
||||
if len(values) == 0 {
|
||||
values = append(values, domain.CaptureDirectoryFromArtifactValue(row.Subject.ItemKey))
|
||||
domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values)
|
||||
if row.PoolSearchCurrent() {
|
||||
values = append(values, row.PoolSearch.PayloadDirectories...)
|
||||
} else {
|
||||
domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values)
|
||||
}
|
||||
}
|
||||
for _, v := range []string{task.ArtifactID, task.SourceVideo} {
|
||||
values = append(values, domain.CaptureDirectoryFromArtifactValue(v))
|
||||
|
||||
@ -50,6 +50,7 @@ func annotationSafeRunUpdate(stored domain.RunRecord, unset []string) (mongo.Pip
|
||||
if e = bson.Unmarshal(raw, &fields); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
stampPoolSearchMetadata("runs", fields)
|
||||
set := bson.M{}
|
||||
for key, value := range fields {
|
||||
set[key] = bson.M{"$literal": value}
|
||||
|
||||
@ -271,6 +271,7 @@ func (store *MongoStore) upsertCaptureSupplierGuarded(ctx context.Context, item
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stampPoolSearchMetadata("captureRecords", set)
|
||||
update := bson.M{"$set": set}
|
||||
if unset := captureRecordUnsetFields(item); len(unset) > 0 {
|
||||
for field := range unset {
|
||||
|
||||
@ -85,6 +85,12 @@ func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
|
||||
func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer cancel()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if q.IDs != nil && len(q.IDs) == 0 || q.AvailableProjectIDs != nil && len(q.AvailableProjectIDs) == 0 {
|
||||
return []domain.HumanItemRecord{}, nil
|
||||
}
|
||||
filter := humanItemMongoFilter(org, q)
|
||||
if q.BlindAnnotationActorID != "" {
|
||||
visibility, err := s.blindAnnotationMongoVisibility(ctx, org, q)
|
||||
@ -93,6 +99,13 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma
|
||||
}
|
||||
filter = bson.M{"$and": bson.A{filter, visibility}}
|
||||
}
|
||||
candidates, err := s.humanPoolSearchCandidates(ctx, org, q, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if candidates != nil {
|
||||
filter = bson.M{"$and": bson.A{filter, candidates}}
|
||||
}
|
||||
pipeline := humanPoolSearchPipeline(org, q, filter)
|
||||
for _, text := range []string{q.Search, q.Capture} {
|
||||
if text != "" {
|
||||
|
||||
@ -13,7 +13,7 @@ import (
|
||||
func humanPoolSearchPipeline(org string, q domain.HumanItemQuery, filter bson.M) mongo.Pipeline {
|
||||
// Only search/category metadata participates in global sorts and joins.
|
||||
// Hydrate matched IDs afterwards; the public record shape stays unchanged.
|
||||
projection := bson.M{"_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1}
|
||||
projection := bson.M{"version": 1, "updatedAt": 1, "poolSearch": 1, "_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1}
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: projection}}}
|
||||
pipeline = append(pipeline,
|
||||
// Number candidate batches; restore the stable createdAt/id order below.
|
||||
|
||||
@ -24,7 +24,7 @@ func (s *MongoStore) humanPoolSearchLegacyPayloads(ctx context.Context, org stri
|
||||
name = domain.CaptureDirectoryFromDirectoryValue(row.Captures[0].Prefix)
|
||||
}
|
||||
}
|
||||
if name == "" {
|
||||
if name == "" && !row.PoolSearchCurrent() {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
}
|
||||
|
||||
@ -25,7 +25,10 @@ func TestHumanPoolSearchMongoLargeModern(t *testing.T) {
|
||||
|
||||
func TestHumanPoolSearchMongoLargePayload(t *testing.T) { testHumanPoolSearchMongo(t, 420, true, true) }
|
||||
|
||||
func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) {
|
||||
func TestHumanPoolSearchMongoReadModel(t *testing.T) {
|
||||
testHumanPoolSearchMongo(t, 10000, true, false, true)
|
||||
}
|
||||
func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool, migrate ...bool) {
|
||||
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
|
||||
if uri == "" {
|
||||
t.Skip("isolated local Mongo opt-in")
|
||||
@ -197,6 +200,20 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) {
|
||||
if _, err := s.annotationTasks.InsertOne(ctx, foreignTask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(migrate) > 0 && migrate[0] {
|
||||
dry, err := s.BackfillPoolSearchMetadata(ctx, org, false)
|
||||
if err != nil || dry.Scanned == 0 || dry.Updated != 0 {
|
||||
t.Fatalf("dry run: %+v %v", dry, err)
|
||||
}
|
||||
result, err := s.BackfillPoolSearchMetadata(ctx, org, true)
|
||||
if err != nil || result.Updated != dry.Scanned || result.Conflicts != 0 || result.Unsupported != 0 {
|
||||
t.Fatalf("backfill: %+v %v", result, err)
|
||||
}
|
||||
again, err := s.BackfillPoolSearchMetadata(ctx, org, true)
|
||||
if err != nil || again.Scanned != 0 {
|
||||
t.Fatalf("idempotence: %+v %v", again, err)
|
||||
}
|
||||
}
|
||||
ids := func(t *testing.T, rows []domain.HumanItemRecord) []string {
|
||||
out := []string{}
|
||||
for _, i := range rows {
|
||||
@ -281,4 +298,47 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) {
|
||||
}
|
||||
})
|
||||
}
|
||||
if len(migrate) > 0 && migrate[0] {
|
||||
item, err := s.GetHumanItem(ctx, "i-0008")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item.Version++
|
||||
item.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
if _, err = s.humanItems.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": bson.M{"subject.payload.importManifest.sourceDirectory": "archive/moved-legacy-capture", "version": item.Version, "updatedAt": item.UpdatedAt}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: "moved-legacy-capture", Limit: 200})
|
||||
if err != nil || len(got) != 1 || got[0].ID != item.ID {
|
||||
t.Fatalf("old writer compatibility: %v %v", ids(t, got), err)
|
||||
}
|
||||
result, err := s.BackfillPoolSearchMetadata(ctx, org, true)
|
||||
if err != nil || result.Updated != 1 {
|
||||
t.Fatalf("repair stale summary: %+v %v", result, err)
|
||||
}
|
||||
current, err := s.GetHumanItem(ctx, item.ID)
|
||||
if err != nil || !current.PoolSearchCurrent() {
|
||||
t.Fatalf("repaired summary not current: %v", err)
|
||||
}
|
||||
next := *current
|
||||
next.Version++
|
||||
next.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
next.Status = domain.HumanItemAccepted
|
||||
changed, err := s.UpdateHumanItemForReview(ctx, next, current.Version)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("review update: %v %v", changed, err)
|
||||
}
|
||||
current, err = s.GetHumanItem(ctx, item.ID)
|
||||
if err != nil || !current.PoolSearchCurrent() || current.Subject.Payload["importManifest"] == nil {
|
||||
t.Fatalf("review lost summary or annotation: %v", err)
|
||||
}
|
||||
if _, err = s.projects.UpdateOne(ctx, bson.M{"id": "p"}, bson.M{"$set": bson.M{"name": "renamed-project"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err = s.ListHumanItems(ctx, org, domain.HumanItemQuery{Search: "renamed-project", Limit: 200, IdentityProjection: true})
|
||||
if err != nil || len(got) != 200 {
|
||||
t.Fatalf("legacy project rename: count=%d error=%v", len(got), err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -529,6 +529,7 @@ func (store *MongoStore) UpsertProject(ctx context.Context, item domain.ProjectR
|
||||
return err
|
||||
}
|
||||
persisted := projectPersistenceDocument(item)
|
||||
stampPoolSearchMetadata("projects", persisted)
|
||||
_, err = store.projects.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": persisted}, options.Update().SetUpsert(allow))
|
||||
return err
|
||||
}
|
||||
|
||||
61
internal/store/pool_search_backfill.go
Normal file
61
internal/store/pool_search_backfill.go
Normal file
@ -0,0 +1,61 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type PoolSearchBackfillResult struct{ Scanned, Updated, Conflicts, Unsupported int64 }
|
||||
|
||||
// BackfillPoolSearchMetadata is an explicit maintenance operation, never called
|
||||
// from GETs or startup. A full-document CAS prevents overwriting any concurrent
|
||||
// writer, including legacy writers without versions/timestamps. Retry conflicts
|
||||
// by rerunning; successful records are skipped. No indexes or business fields
|
||||
// are changed. Dry runs are read-only.
|
||||
func (s *MongoStore) BackfillPoolSearchMetadata(ctx context.Context, org string, apply bool) (PoolSearchBackfillResult, error) {
|
||||
result := PoolSearchBackfillResult{}
|
||||
for _, name := range []string{"humanItems", "runs", "annotationTasks", "captureRecords", "projects"} {
|
||||
collection := s.db.Collection(name)
|
||||
filter := bson.M{"organizationId": normalizedOrganizationID(org), "$expr": bson.M{"$not": bson.A{poolSearchValidExpression(name)}}}
|
||||
cursor, err := collection.Find(ctx, filter, options.Find().SetBatchSize(16))
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
err = func() error {
|
||||
defer cursor.Close(ctx)
|
||||
for cursor.Next(ctx) {
|
||||
original := append(bson.Raw(nil), cursor.Current...)
|
||||
var d bson.M
|
||||
if err := bson.Unmarshal(original, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
result.Scanned++
|
||||
stampPoolSearchMetadata(name, d)
|
||||
metadata := d[poolSearchField]
|
||||
if metadata == nil {
|
||||
result.Unsupported++
|
||||
continue
|
||||
}
|
||||
if !apply {
|
||||
continue
|
||||
}
|
||||
update, err := collection.UpdateOne(ctx, bson.M{"_id": d["_id"], "$expr": bson.M{"$eq": bson.A{"$$ROOT", bson.M{"$literal": original}}}}, bson.M{"$set": bson.M{poolSearchField: metadata}})
|
||||
if err != nil {
|
||||
return fmt.Errorf("backfill %s: %w", name, err)
|
||||
}
|
||||
if update.MatchedCount == 0 {
|
||||
result.Conflicts++
|
||||
} else {
|
||||
result.Updated++
|
||||
}
|
||||
}
|
||||
return cursor.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
330
internal/store/pool_search_metadata.go
Normal file
330
internal/store/pool_search_metadata.go
Normal file
@ -0,0 +1,330 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// Owned by the source document, updated in the same write after supplier
|
||||
// scrubbing. This contains names only, never permissions, eligibility decisions or customer text.
|
||||
const poolSearchSchema = 1
|
||||
const poolSearchField = "poolSearch"
|
||||
|
||||
type poolSearchMetadata = domain.HumanPoolSearchMetadata
|
||||
|
||||
func stampPoolSearchMetadata(collection string, d bson.M) {
|
||||
m := poolSearchMetadata{Source: map[string]any{}, Schema: poolSearchSchema, Text: []string{}, PayloadDirectories: []string{}}
|
||||
add := func(keys ...string) {
|
||||
for _, k := range keys {
|
||||
if v, ok := d[k].(string); ok {
|
||||
m.Text = append(m.Text, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch collection {
|
||||
case "humanItems":
|
||||
add("id", "legacyTaskId")
|
||||
subject := adminPoolMap(d["subject"])
|
||||
for _, k := range []string{"itemKey", "artifactId"} {
|
||||
if v, ok := subject[k].(string); ok {
|
||||
m.Text = append(m.Text, v)
|
||||
}
|
||||
}
|
||||
domain.AppendCaptureDirectoriesFromPayload(subject["payload"], &m.PayloadDirectories)
|
||||
for _, value := range deletionArray(deletionDoc(subject["payload"])["items"]) {
|
||||
coverage := deletionDoc(deletionDoc(deletionDoc(value)["materialization"])["coverage"])
|
||||
var seconds float64
|
||||
switch n := coverage["durationSeconds"].(type) {
|
||||
case float64:
|
||||
seconds = n
|
||||
case int:
|
||||
seconds = float64(n)
|
||||
case int32:
|
||||
seconds = float64(n)
|
||||
case int64:
|
||||
seconds = float64(n)
|
||||
}
|
||||
if seconds > 0 && !math.IsInf(seconds, 0) && !math.IsNaN(seconds) {
|
||||
m.DurationSeconds += seconds
|
||||
}
|
||||
}
|
||||
m.Text = append(m.Text, m.PayloadDirectories...)
|
||||
case "annotationTasks":
|
||||
add("artifactId", "sourceVideo")
|
||||
case "captureRecords":
|
||||
add("captureId", "prefix")
|
||||
case "projects":
|
||||
add("name")
|
||||
case "runs":
|
||||
// Decode only the fields used by the existing directory parser.
|
||||
raw, err := bson.Marshal(poolRunSearchDocument(d))
|
||||
var run domain.RunRecord
|
||||
if err != nil || bson.Unmarshal(raw, &run) != nil {
|
||||
d[poolSearchField] = nil
|
||||
return
|
||||
}
|
||||
m.Text = append(m.Text, domain.CaptureDirectoriesFromRun(run)...)
|
||||
add("projectName")
|
||||
default:
|
||||
return
|
||||
}
|
||||
normalize := func(values []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := []string{}
|
||||
for _, v := range values {
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
if v != "" && !seen[v] {
|
||||
seen[v] = true
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
for _, value := range append([]string(nil), m.Text...) {
|
||||
m.Text = append(m.Text, domain.CaptureDirectoryFromArtifactValue(value), domain.CaptureDirectoryFromDirectoryValue(value))
|
||||
}
|
||||
m.Text = normalize(m.Text)
|
||||
for n, path := range poolSearchSourceFields(collection, d) {
|
||||
m.Source[fmt.Sprint(n)] = poolSearchSourceValue(d, path)
|
||||
}
|
||||
if collection == "humanItems" {
|
||||
m.SourceVersion = int(supplierNumber(d["version"]))
|
||||
m.SourceUpdatedAt, _ = d["updatedAt"].(string)
|
||||
subject := adminPoolMap(d["subject"])
|
||||
m.SourceItemKey, _ = subject["itemKey"].(string)
|
||||
m.SourceArtifactID, _ = subject["artifactId"].(string)
|
||||
}
|
||||
// Preserve the exact historical values/order for the final matcher. Only
|
||||
// prefilter text is normalized; the matcher owns display precedence.
|
||||
encoded, err := bson.Marshal(m)
|
||||
if err != nil || len(encoded) > 64<<10 {
|
||||
d[poolSearchField] = nil
|
||||
return
|
||||
}
|
||||
d[poolSearchField] = m
|
||||
}
|
||||
|
||||
// A conservative reverse lookup narrows candidates BEFORE joins/sorts. Missing
|
||||
// summaries always enter the old path; the final matcher and live permissions
|
||||
// remain authoritative. Very broad reference sets use the bounded old pipeline
|
||||
// rather than truncating a result or building an unbounded Mongo command.
|
||||
func (s *MongoStore) humanPoolSearchCandidates(ctx context.Context, org string, q domain.HumanItemQuery, base bson.M) (bson.M, error) {
|
||||
text := q.Capture
|
||||
if text == "" {
|
||||
text = q.Search
|
||||
}
|
||||
words := strings.Fields(text)
|
||||
if len(words) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
// Do not fetch thousands of foreign legacy IDs just to discover that the
|
||||
// original pipeline is required. A tiny readiness probe preserves the legacy
|
||||
// no-hit wire budget and keeps mixed-version reads on the proven path.
|
||||
legacy := bson.M{"$and": bson.A{base, bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression("humanItems")}}}}}
|
||||
err := s.humanItems.FindOne(ctx, legacy, options.FindOne().SetProjection(bson.M{"_id": 1}).SetMaxTime(8*time.Second)).Err()
|
||||
if err == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if err != mongo.ErrNoDocuments {
|
||||
return nil, err
|
||||
}
|
||||
token := strings.ToLower(words[0])
|
||||
if i := strings.LastIndex(token, "/part"); i > 0 {
|
||||
token = token[:i]
|
||||
}
|
||||
pattern := regexp.QuoteMeta(token) + "|%[0-9a-f]{2}"
|
||||
matchFor := func(collection string) bson.M {
|
||||
return bson.M{"$or": bson.A{bson.M{"$expr": bson.M{"$not": bson.A{poolSearchValidExpression(collection)}}}, bson.M{poolSearchField + ".text": bson.M{"$regex": pattern}}}}
|
||||
}
|
||||
match := matchFor("captureRecords")
|
||||
const maxReferences = 4096
|
||||
ids := func(collection string, extra bson.M) ([]string, bool, error) {
|
||||
f := bson.M{"organizationId": normalizedOrganizationID(org), "$and": bson.A{match}}
|
||||
if extra != nil {
|
||||
f["$and"] = append(f["$and"].(bson.A), extra)
|
||||
}
|
||||
cur, err := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetLimit(maxReferences+1).SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
out := []string{}
|
||||
for cur.Next(ctx) {
|
||||
var v struct {
|
||||
ID string `bson:"id"`
|
||||
}
|
||||
if err := cur.Decode(&v); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
out = append(out, v.ID)
|
||||
}
|
||||
return out, len(out) > maxReferences, cur.Err()
|
||||
}
|
||||
captureIDs, broad, err := ids("captureRecords", nil)
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
// A capture match follows only same-organization runs. Tenant checks in the
|
||||
// existing join are retained even after this candidate narrowing.
|
||||
runMatch := matchFor("runs")
|
||||
if len(captureIDs) > 0 {
|
||||
runMatch = bson.M{"$or": bson.A{runMatch, bson.M{"taskId": bson.M{"$in": captureIDs}}}}
|
||||
}
|
||||
match = runMatch
|
||||
runIDs, broad, err := ids("runs", nil)
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
match = matchFor("annotationTasks")
|
||||
taskIDs, broad, err := ids("annotationTasks", nil)
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
match = matchFor("projects")
|
||||
projectIDs, broad, err := ids("projects", nil)
|
||||
if err != nil || broad {
|
||||
return nil, err
|
||||
}
|
||||
branches := bson.A{matchFor("humanItems")}
|
||||
for field, values := range map[string][]string{"runId": runIDs, "legacyTaskId": taskIDs, "projectId": projectIDs} {
|
||||
if len(values) > 0 {
|
||||
branches = append(branches, bson.M{field: bson.M{"$in": values}})
|
||||
}
|
||||
}
|
||||
return bson.M{"$or": branches}, nil
|
||||
}
|
||||
|
||||
func poolSearchSourceValue(d bson.M, path string) any {
|
||||
if path == "__runReferences" {
|
||||
return poolRunReferences(d)
|
||||
}
|
||||
parts := strings.Split(path, ".")
|
||||
for _, part := range parts[:len(parts)-1] {
|
||||
d = deletionDoc(d[part])
|
||||
}
|
||||
return d[parts[len(parts)-1]]
|
||||
}
|
||||
func poolSearchSourceFields(collection string, d bson.M) []string {
|
||||
switch collection {
|
||||
case "humanItems":
|
||||
return []string{"version", "updatedAt", "subject.itemKey", "subject.artifactId"}
|
||||
case "projects":
|
||||
return []string{"name"}
|
||||
case "annotationTasks":
|
||||
return []string{"artifactId", "sourceVideo"}
|
||||
case "captureRecords":
|
||||
return []string{"captureId", "prefix"}
|
||||
case "runs":
|
||||
return []string{"projectName", "captureDirectories", "__runReferences"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func poolSearchValidExpression(collection string) bson.M {
|
||||
fields := poolSearchSourceFields(collection, nil)
|
||||
terms := bson.A{bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$poolSearch.schema", 0}}, poolSearchSchema}}}
|
||||
for n, path := range fields {
|
||||
terms = append(terms, bson.M{"$eq": bson.A{poolSearchSourceExpression(path), bson.M{"$ifNull": bson.A{"$poolSearch.source." + fmt.Sprint(n), nil}}}})
|
||||
}
|
||||
return bson.M{"$and": terms}
|
||||
}
|
||||
|
||||
func poolSearchSourceExpression(path string) any {
|
||||
if path != "__runReferences" {
|
||||
return bson.M{"$ifNull": bson.A{"$" + path, nil}}
|
||||
}
|
||||
text := func(value string) bson.M { return bson.M{"$ifNull": bson.A{value, ""}} }
|
||||
names := func(input string) bson.M {
|
||||
return bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "as": "artifact", "in": bson.A{text("$$artifact.id"), text("$$artifact.name"), text("$$artifact.path")}}}
|
||||
}
|
||||
inputs := bson.M{"$map": bson.M{"input": bson.M{"$sortArray": bson.M{"input": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$inputs", bson.M{}}}}, "sortBy": bson.M{"k": 1}}}, "as": "input", "in": bson.A{"$$input.k", text("$$input.v.ref.captureUnit"), text("$$input.v.ref.prefix"), text("$$input.v.ref.key"), text("$$input.v.ref.manifestKey")}}}
|
||||
nodes := bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "node", "in": names("$$node.artifacts")}}
|
||||
return bson.M{"$cond": bson.A{bson.M{"$gt": bson.A{bson.M{"$size": bson.M{"$ifNull": bson.A{"$captureDirectories", bson.A{}}}}, 0}}, bson.A{}, bson.A{inputs, names("$artifacts"), nodes}}}
|
||||
}
|
||||
func poolRunReferences(d bson.M) []any {
|
||||
if len(deletionArray(d["captureDirectories"])) > 0 {
|
||||
return []any{}
|
||||
}
|
||||
text := func(d bson.M, key string) string { v, _ := d[key].(string); return v }
|
||||
names := func(value any) []any {
|
||||
out := []any{}
|
||||
for _, v := range deletionArray(value) {
|
||||
a := deletionDoc(v)
|
||||
out = append(out, []any{text(a, "id"), text(a, "name"), text(a, "path")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
inputs := deletionDoc(d["inputs"])
|
||||
keys := []string{}
|
||||
for k := range inputs {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
refs := []any{}
|
||||
for _, k := range keys {
|
||||
ref := deletionDoc(deletionDoc(inputs[k])["ref"])
|
||||
refs = append(refs, []any{k, text(ref, "captureUnit"), text(ref, "prefix"), text(ref, "key"), text(ref, "manifestKey")})
|
||||
}
|
||||
nodes := []any{}
|
||||
for _, v := range deletionArray(d["nodeExecutions"]) {
|
||||
nodes = append(nodes, names(deletionDoc(v)["artifacts"]))
|
||||
}
|
||||
return []any{refs, names(d["artifacts"]), nodes}
|
||||
}
|
||||
|
||||
func poolRunSearchDocument(d bson.M) bson.M {
|
||||
names := func(value any) []any {
|
||||
out := []any{}
|
||||
for _, value := range deletionArray(value) {
|
||||
row := deletionDoc(value)
|
||||
out = append(out, bson.M{"id": row["id"], "name": row["name"], "path": row["path"]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
inputs := bson.M{}
|
||||
for key, value := range deletionDoc(d["inputs"]) {
|
||||
ref := deletionDoc(deletionDoc(value)["ref"])
|
||||
selected := bson.M{}
|
||||
for _, field := range []string{"captureUnit", "prefix", "key", "manifestKey"} {
|
||||
if v, ok := ref[field]; ok {
|
||||
selected[field] = v
|
||||
}
|
||||
}
|
||||
inputs[key] = bson.M{"ref": selected}
|
||||
}
|
||||
nodes := []any{}
|
||||
for _, value := range deletionArray(d["nodeExecutions"]) {
|
||||
nodes = append(nodes, bson.M{"artifacts": names(deletionDoc(value)["artifacts"])})
|
||||
}
|
||||
return bson.M{"captureDirectories": d["captureDirectories"], "inputs": inputs, "artifacts": names(d["artifacts"]), "nodeExecutions": nodes}
|
||||
}
|
||||
|
||||
// State-only review writes must keep a valid name summary current without
|
||||
// re-reading/rewriting customer annotation payloads. Invalid summaries remain
|
||||
// invalid and take the compatibility path. The version CAS fences this update.
|
||||
func poolSearchStateUpdate(fields bson.M) mongo.Pipeline {
|
||||
set := bson.M{}
|
||||
for key, value := range fields {
|
||||
set[key] = bson.M{"$literal": value}
|
||||
}
|
||||
set[poolSearchField] = bson.M{"$cond": bson.A{
|
||||
poolSearchValidExpression("humanItems"),
|
||||
bson.M{"$mergeObjects": bson.A{"$poolSearch", bson.M{
|
||||
"sourceVersion": bson.M{"$literal": fields["version"]},
|
||||
"sourceUpdatedAt": bson.M{"$literal": fields["updatedAt"]},
|
||||
"source": bson.M{"$mergeObjects": bson.A{"$poolSearch.source", bson.M{"0": bson.M{"$literal": fields["version"]}, "1": bson.M{"$literal": fields["updatedAt"]}}}},
|
||||
}}},
|
||||
"$poolSearch",
|
||||
}}
|
||||
return mongo.Pipeline{bson.D{{Key: "$set", Value: set}}}
|
||||
}
|
||||
43
internal/store/pool_search_metadata_test.go
Normal file
43
internal/store/pool_search_metadata_test.go
Normal file
@ -0,0 +1,43 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPoolSearchMetadataOwnsOnlyNames(t *testing.T) {
|
||||
doc := supplierDoc(domain.HumanItemRecord{ID: "i", Subject: domain.HumanSubjectRef{ItemKey: "archive/目录%20甲/part0002.mp4", Payload: map[string]any{"private": "secret comment", "nested": map[string]any{"sourceDirectory": "archive/old-capture"}}}})
|
||||
stampPoolSearchMetadata("humanItems", doc)
|
||||
m := doc[poolSearchField].(poolSearchMetadata)
|
||||
if !reflect.DeepEqual(m.PayloadDirectories, []string{"old-capture"}) {
|
||||
t.Fatal(m)
|
||||
}
|
||||
for _, v := range m.Text {
|
||||
if v == "secret comment" {
|
||||
t.Fatal("customer text indexed")
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, v := range m.Text {
|
||||
if v == "archive/目录%20甲/part0002.mp4" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("encoded artifact compatibility", m)
|
||||
}
|
||||
row := adminPoolRow{HumanItemRecord: domain.HumanItemRecord{PoolSearch: &m, Subject: domain.HumanSubjectRef{ItemKey: m.SourceItemKey}}}
|
||||
if !humanPoolSearchRowMatches(row, domain.HumanItemQuery{Capture: "old-capture"}) {
|
||||
t.Fatal("summary lost legacy directory")
|
||||
}
|
||||
// Authoritative capture names keep their precedence over old payload names.
|
||||
row.Captures = []domain.CaptureRecord{{CaptureID: "authoritative"}}
|
||||
if humanPoolSearchRowMatches(row, domain.HumanItemQuery{Capture: "old-capture"}) {
|
||||
t.Fatal("payload overrode capture")
|
||||
}
|
||||
if _, err := bson.Marshal(doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -95,7 +95,7 @@ func humanReviewFields(v domain.HumanItemRecord) bson.M {
|
||||
"reviewOperationId": v.ReviewOperationID}
|
||||
}
|
||||
func (s *MongoStore) UpdateHumanItemForReview(ctx context.Context, next domain.HumanItemRecord, expected int) (bool, error) {
|
||||
r, err := s.humanItems.UpdateOne(ctx, bson.M{"id": next.ID, "organizationId": next.OrganizationID, "version": expected}, bson.M{"$set": humanReviewFields(next)})
|
||||
r, err := s.humanItems.UpdateOne(ctx, bson.M{"id": next.ID, "organizationId": next.OrganizationID, "version": expected}, poolSearchStateUpdate(humanReviewFields(next)))
|
||||
return err == nil && r.MatchedCount == 1, err
|
||||
}
|
||||
func (s *MemoryStore) UpdateHumanItemForReview(_ context.Context, next domain.HumanItemRecord, expected int) (bool, error) {
|
||||
|
||||
@ -162,12 +162,18 @@ func scrubSupplierDocument(d bson.M, p domain.SupplierDeletionPlan) bool {
|
||||
delete(d, "vendorId")
|
||||
}
|
||||
if changed {
|
||||
delete(d, poolSearchField)
|
||||
d["updatedAt"] = nowISO()
|
||||
}
|
||||
return changed
|
||||
}
|
||||
func supplierGuardedMongoWrite(ctx context.Context, col *mongo.Collection, id string, item any, write func(context.Context, bson.M) error) error {
|
||||
d := supplierDoc(item)
|
||||
writeSource := write
|
||||
write = func(ctx context.Context, document bson.M) error {
|
||||
stampPoolSearchMetadata(col.Name(), document)
|
||||
return writeSource(ctx, document)
|
||||
}
|
||||
if !supplierGuardRelevant(col.Name()) {
|
||||
return write(ctx, d)
|
||||
}
|
||||
|
||||
27
internal/workforcestats/report_location.go
Normal file
27
internal/workforcestats/report_location.go
Normal file
@ -0,0 +1,27 @@
|
||||
package workforcestats
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var reportLocations = struct {
|
||||
sync.Mutex
|
||||
values map[string]*time.Location
|
||||
}{values: map[string]*time.Location{}}
|
||||
|
||||
// ReportingLocation shares immutable IANA rules, with bounded retention. This
|
||||
// makes location identity stable across equivalent report queries without ever
|
||||
// treating two arbitrary custom time.Location values as equivalent.
|
||||
func ReportingLocation(name string) (*time.Location, error) {
|
||||
reportLocations.Lock()
|
||||
defer reportLocations.Unlock()
|
||||
if location := reportLocations.values[name]; location != nil {
|
||||
return location, nil
|
||||
}
|
||||
location, err := time.LoadLocation(name)
|
||||
if err == nil && len(reportLocations.values) < 32 {
|
||||
reportLocations.values[name] = location
|
||||
}
|
||||
return location, err
|
||||
}
|
||||
@ -434,6 +434,53 @@
|
||||
"tests": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "query-readmodels",
|
||||
"paths": [
|
||||
"^internal/store/pool_search.*\\.go$",
|
||||
"^cmd/pool-search-backfill/",
|
||||
"^internal/store/(mongo|annotation_artifact|capture_automation|supplier_deletion_guard|review_decision)\\.go$",
|
||||
"^internal/api/(human_display|production_quality|routes_human_loop)\\.go$",
|
||||
"^internal/api/testdata/routes\\.golden$",
|
||||
"^internal/api/poll_pool_load_test\\.go$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {
|
||||
"./internal/api": ["^TestHumanPoolSummary", "^TestHumanPoolCounts", "^TestHumanDisplay", "^TestRouteSnapshot$"],
|
||||
"./internal/store": ["^TestPoolSearchMetadata"]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
"argv": ["npm", "run", "test:go:gate"],
|
||||
"tests": true
|
||||
},
|
||||
{
|
||||
"argv": ["go", "test", "./cmd/pool-search-backfill", "-count=1"]
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"go",
|
||||
"test",
|
||||
"./internal/api",
|
||||
"-run",
|
||||
"^TestPollPoolReadMixLocal$",
|
||||
"-count=1",
|
||||
"-timeout=3m",
|
||||
"-v"
|
||||
],
|
||||
"env": {
|
||||
"POLL_POOL_LOAD": "1",
|
||||
"POLL_POOL_READMODELS": "1",
|
||||
"POLL_POOL_SEARCH": "1",
|
||||
"POLL_POOL_AUTHORIZATION_STATE": "initialized",
|
||||
"POLL_POOL_WARMUP_SECONDS": "5",
|
||||
"POLL_POOL_DURATION_SECONDS": "30"
|
||||
},
|
||||
"tests": true
|
||||
}
|
||||
],
|
||||
"warnings": ["本地 81 人读取冒烟不代表持续容量验收;摘要补建生产 apply 需单独授权,旧记录兼容路径须保留。"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { isHumanPoolSummary } from "../types";
|
||||
import { ReviewListTrace } from "./ReviewTrace";
|
||||
import type { HumanItem } from "../types";
|
||||
import { ReviewHistorySummary } from "./ReviewHistorySummary";
|
||||
@ -32,10 +33,13 @@ export function AcceptanceTaskCard({
|
||||
const completed = ["accepted", "released"].includes(item.status);
|
||||
const reworkReview = !completed && item.status !== "rejected" && Number(item.reviewRound || 0) > 0;
|
||||
const entries = Array.isArray(item.subject?.payload?.items) ? item.subject.payload.items : [];
|
||||
const seconds = entries.reduce(
|
||||
(total: number, entry: any) => total + Math.max(0, Number(entry.materialization?.coverage?.durationSeconds) || 0),
|
||||
0,
|
||||
);
|
||||
const seconds = isHumanPoolSummary(item)
|
||||
? item.durationSeconds || 0
|
||||
: entries.reduce(
|
||||
(total: number, entry: any) =>
|
||||
total + Math.max(0, Number(entry.materialization?.coverage?.durationSeconds) || 0),
|
||||
0,
|
||||
);
|
||||
const action = acceptanceTaskAction({ status: item.status, mine, claimable });
|
||||
return (
|
||||
<article
|
||||
|
||||
@ -1,3 +1,18 @@
|
||||
// Only route absence permits a rolling-upgrade fallback. Summary objects stay
|
||||
// marked as summaries; selected workbenches fetch detail independently.
|
||||
export async function loadHumanPoolList(human, filters, options) {
|
||||
if (human.listSummariesPage) {
|
||||
try {
|
||||
const page = await human.listSummariesPage(filters, options);
|
||||
if (!Array.isArray(page.summaries)) throw new Error("任务列表响应不完整,请重试。");
|
||||
return { ...page, items: page.summaries };
|
||||
} catch (error) {
|
||||
if (error?.status !== 404 && error?.status !== 405) throw error;
|
||||
}
|
||||
}
|
||||
return human.listItemsPage(filters, options);
|
||||
}
|
||||
|
||||
// Keep GET joins below proxy request-line limits, with bounded concurrency.
|
||||
export async function loadAnnotationTaskJoin(annotation, filters, ids, signal = undefined) {
|
||||
signal?.throwIfAborted();
|
||||
@ -131,7 +146,8 @@ export async function loadAnnotationPoolPage({ annotation, human, scope, view, c
|
||||
: ["available", "mine", "reviewed"];
|
||||
const pages = await Promise.all(
|
||||
names.map(async (pool) => {
|
||||
const page = await human.listItemsPage(
|
||||
const page = await loadHumanPoolList(
|
||||
human,
|
||||
{
|
||||
pluginId: "annotation-qa",
|
||||
projectId: scope.projectId,
|
||||
|
||||
11
src/lib/review-item-selection.js
Normal file
11
src/lib/review-item-selection.js
Normal file
@ -0,0 +1,11 @@
|
||||
// List summaries select an identity, never editable content. Keep legacy
|
||||
// annotation/package detail fallback when no aggregate pool item is selected.
|
||||
export function reviewItemForSelection(poolItem, aggregateDetail, annotationItem) {
|
||||
if (!poolItem) return annotationItem;
|
||||
if (poolItem.kind !== "pool-summary") return poolItem;
|
||||
return aggregateDetail?.id === poolItem.id &&
|
||||
aggregateDetail.version >= poolItem.version &&
|
||||
aggregateDetail.kind !== "pool-summary"
|
||||
? aggregateDetail
|
||||
: undefined;
|
||||
}
|
||||
@ -1,3 +1,5 @@
|
||||
import { reviewItemForSelection } from "../lib/review-item-selection.js";
|
||||
import { isHumanPoolSummary } from "../types";
|
||||
import { annotationTimelineFailure } from "../lib/annotation-source-validation.js";
|
||||
import type { AnnotationTimelineFailure } from "../lib/annotation-source-validation";
|
||||
// Keep the workbench rules ahead of the review framing rules, matching the
|
||||
@ -749,6 +751,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
const [templates, setTemplates] = useState<AnnotationTemplate[]>([]);
|
||||
const [tasks, setTasks] = useState<AnnotationTaskRecord[]>([]);
|
||||
const [humanPools, setHumanPools] = useState<HumanPools>(emptyHumanPools);
|
||||
const [reviewPoolTotals, setReviewPoolTotals] = useState<{
|
||||
scope: string;
|
||||
counts: Record<TaskPoolView, number>;
|
||||
} | null>(null);
|
||||
const [reviewPoolCountError, setReviewPoolCountError] = useState("");
|
||||
const [acceptancePoolDetail, setAcceptancePoolDetail] = useState<{ scope: string; item: HumanItem } | null>(null);
|
||||
const [acceptancePoolDetailError, setAcceptancePoolDetailError] = useState("");
|
||||
const [acceptanceDetailRetry, setAcceptanceDetailRetry] = useState(0);
|
||||
// A URL task id is a selection, not ownership; deep-link previews retain their pool.
|
||||
const defaultTaskPoolView = () =>
|
||||
initialAnnotationPool({
|
||||
@ -1065,6 +1075,39 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
...(managerAnnotationPool ? [managerPoolFilters] : []),
|
||||
...(pagedReviewPool ? [adminSearch, taskStatusFilter, requestedCaptureDirectory] : []),
|
||||
]);
|
||||
useEffect(() => {
|
||||
if (!pagedReviewPool) return;
|
||||
const controller = new AbortController();
|
||||
setReviewPoolCountError("");
|
||||
void humanLoopApi
|
||||
.poolCounts(
|
||||
{
|
||||
pluginId: "annotation-qa",
|
||||
projectId: projectId || undefined,
|
||||
workflowId: requestedWorkflowId || undefined,
|
||||
runId: requestedRunId || undefined,
|
||||
search: adminSearch || undefined,
|
||||
capture: requestedCaptureDirectory || undefined,
|
||||
category: taskStatusFilter,
|
||||
stageIds: acceptanceWorkspace
|
||||
? ["acceptance", "prepackage_acceptance"]
|
||||
: humanItemStageIdsForMode(workspaceMode),
|
||||
reviewedStageIds: acceptanceWorkspace ? ["acceptance", "prepackage_acceptance"] : undefined,
|
||||
},
|
||||
{ signal: controller.signal },
|
||||
)
|
||||
.then((result) => {
|
||||
if (!controller.signal.aborted) setReviewPoolTotals({ scope: projectDataScope, counts: result.counts });
|
||||
})
|
||||
.catch((cause) => {
|
||||
if (!controller.signal.aborted) {
|
||||
setReviewPoolTotals(null);
|
||||
if (cause?.status !== 404 && cause?.status !== 405)
|
||||
setReviewPoolCountError(getActionErrorMessage(cause, "加载任务数量失败"));
|
||||
}
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, [pagedReviewPool, projectDataScope, humanPools]);
|
||||
const reviewPoolViewRef = useRef(taskPoolView);
|
||||
reviewPoolViewRef.current = taskPoolView;
|
||||
const reviewActionScopeRef = useRef(projectDataScope);
|
||||
@ -2137,8 +2180,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
if (prepackageAcceptanceItems.some((item) => item.id === requestedAcceptanceId))
|
||||
setExpandedDeliveryPackageId(requestedAcceptanceId);
|
||||
}, [workspaceMode, requestedAcceptanceId, prepackageAcceptanceItems, loading, saving]);
|
||||
const reviewSessionItem =
|
||||
prepackageAcceptanceItems.find((item) => item.id === expandedDeliveryPackageId) || detail?.humanItem;
|
||||
const acceptanceListSelection = prepackageAcceptanceItems.find((item) => item.id === expandedDeliveryPackageId);
|
||||
const acceptanceDetailSelection =
|
||||
acceptancePoolDetail?.scope === projectDataScope &&
|
||||
acceptancePoolDetail.item.id === acceptanceListSelection?.id &&
|
||||
acceptancePoolDetail.item.version >= acceptanceListSelection.version
|
||||
? acceptancePoolDetail.item
|
||||
: null;
|
||||
useEffect(() => {
|
||||
if (!acceptanceListSelection || !isHumanPoolSummary(acceptanceListSelection)) return;
|
||||
const controller = new AbortController();
|
||||
setAcceptancePoolDetailError("");
|
||||
void humanLoopApi
|
||||
.getItem(acceptanceListSelection.id, { signal: controller.signal })
|
||||
.then(({ item }) => {
|
||||
if (!controller.signal.aborted) setAcceptancePoolDetail({ scope: projectDataScope, item });
|
||||
})
|
||||
.catch((cause) => {
|
||||
if (!controller.signal.aborted) setAcceptancePoolDetailError(getActionErrorMessage(cause, "加载验收内容失败"));
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, [projectDataScope, acceptanceListSelection?.id, acceptanceListSelection?.version, acceptanceDetailRetry]);
|
||||
const reviewSessionItem = reviewItemForSelection(
|
||||
acceptanceListSelection,
|
||||
acceptanceDetailSelection,
|
||||
detail?.humanItem,
|
||||
);
|
||||
const reviewSessionEnabled = Boolean(
|
||||
(workspaceMode === "quality" || workspaceMode === "clip-quality" || workspaceMode === "acceptance") &&
|
||||
taskPoolView !== "reviewed" &&
|
||||
@ -2817,8 +2884,18 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
: { available: 0, mine: 0, all: 0 }),
|
||||
reviewed: 0,
|
||||
}
|
||||
: poolTabCounts(humanPools, matchesPoolScope),
|
||||
[humanPools, matchesPoolScope, managerAnnotationPool, adminPoolTotals, projectDataScope],
|
||||
: pagedReviewPool && reviewPoolTotals?.scope === projectDataScope
|
||||
? reviewPoolTotals.counts
|
||||
: poolTabCounts(humanPools, matchesPoolScope),
|
||||
[
|
||||
humanPools,
|
||||
matchesPoolScope,
|
||||
managerAnnotationPool,
|
||||
adminPoolTotals,
|
||||
projectDataScope,
|
||||
pagedReviewPool,
|
||||
reviewPoolTotals,
|
||||
],
|
||||
);
|
||||
const stageEmptyLabel =
|
||||
workspaceMode === "quality"
|
||||
@ -7043,9 +7120,9 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
aria-pressed={taskPoolView === value}
|
||||
className={`rounded-lg border px-3 py-2 text-xs transition-colors ${taskPoolView === value ? "border-blue-500 bg-blue-50 text-blue-600 dark:bg-blue-500/10 dark:text-blue-200 font-medium" : "border-[color:var(--app-border)] text-slate-600 hover:text-slate-900 dark:text-gray-400 dark:hover:text-white"}`}
|
||||
>
|
||||
{label} {pagedReviewPool ? "已加载 " : ""}
|
||||
{label}
|
||||
{(managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope) ||
|
||||
(pagedReviewPool && !(value in reviewPoolCursorsRef.current))
|
||||
(pagedReviewPool && reviewPoolTotals?.scope !== projectDataScope)
|
||||
? "—"
|
||||
: blindAnnotator && value === "available"
|
||||
? claimableCount.projectId === projectId
|
||||
@ -7053,10 +7130,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
: "—"
|
||||
: count}
|
||||
{/* Counts only loaded pages; a further page means more exist. */}
|
||||
{(pagedReviewPool && reviewPoolCursorsRef.current[value]) ||
|
||||
(blindAnnotator && value === "mine" && minePoolCursorRef.current)
|
||||
? "+"
|
||||
: ""}
|
||||
{blindAnnotator && value === "mine" && minePoolCursorRef.current ? "+" : ""}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
@ -7183,6 +7257,11 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
<div className="annotation-task-list-label mt-4 text-xs font-medium text-slate-500 dark:text-gray-400">
|
||||
逐段任务
|
||||
</div>
|
||||
{reviewPoolCountError && (
|
||||
<div role="alert" className="mt-2 text-xs text-red-600">
|
||||
{reviewPoolCountError}
|
||||
</div>
|
||||
)}
|
||||
<div className="annotation-task-list mt-3 max-h-[calc(100vh-330px)] space-y-1.5 overflow-auto pr-1">
|
||||
{sidebarLoading && (
|
||||
<div role="status" className="p-3 text-sm">
|
||||
@ -7582,7 +7661,27 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
)}
|
||||
{prepackageAcceptanceItems
|
||||
.filter((i) => i.id === expandedDeliveryPackageId)
|
||||
.map((item) => {
|
||||
.map((poolItem) => {
|
||||
const item = isHumanPoolSummary(poolItem) ? acceptanceDetailSelection : poolItem;
|
||||
if (!item)
|
||||
return (
|
||||
<div
|
||||
key={poolItem.id}
|
||||
role={acceptancePoolDetailError ? "alert" : "status"}
|
||||
className="p-3 text-sm"
|
||||
>
|
||||
{acceptancePoolDetailError || "正在加载验收内容…"}
|
||||
{acceptancePoolDetailError && (
|
||||
<button
|
||||
type="button"
|
||||
className="ml-2 underline"
|
||||
onClick={() => setAcceptanceDetailRetry((value) => value + 1)}
|
||||
>
|
||||
重试
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
const stats = prepackageAcceptanceDrafts[item.id] || {
|
||||
...emptyPrepackageAcceptanceStatistics,
|
||||
headpose: "include",
|
||||
|
||||
@ -1,4 +1,10 @@
|
||||
import type { HumanItem, HumanItemStatus, HumanReject, RunArtifactManifestMembersResponse } from "../../types";
|
||||
import type {
|
||||
HumanItem,
|
||||
HumanPoolSummary,
|
||||
HumanItemStatus,
|
||||
HumanReject,
|
||||
RunArtifactManifestMembersResponse,
|
||||
} from "../../types";
|
||||
import { mergePagedManifestMembers } from "../../lib/annotation-workbench.js";
|
||||
import { apiRequest } from "./client";
|
||||
|
||||
@ -89,6 +95,19 @@ async function listAcceptancePool(filters: HumanItemFilters, options: { signal?:
|
||||
export const humanLoopApi = {
|
||||
listItems,
|
||||
listItemsPage,
|
||||
listSummariesPage: (filters: HumanItemFilters = {}, options: { signal?: AbortSignal } = {}) =>
|
||||
apiRequest<{ summaries: HumanPoolSummary[]; filtersApplied: boolean; hasMore: boolean; nextCursor?: string }>(
|
||||
`/api/human-item-summaries${queryString(filters)}`,
|
||||
options,
|
||||
),
|
||||
poolCounts: (
|
||||
filters: HumanItemFilters & { reviewedStageIds?: string[] } = {},
|
||||
options: { signal?: AbortSignal } = {},
|
||||
) =>
|
||||
apiRequest<{ counts: Record<"available" | "mine" | "reviewed" | "all", number>; filtersApplied: boolean }>(
|
||||
`/api/human-item-counts${queryString(filters)}`,
|
||||
options,
|
||||
),
|
||||
listAllItems,
|
||||
listAcceptancePools: async (
|
||||
filters: Omit<HumanItemFilters, "stageId" | "pool">,
|
||||
@ -110,11 +129,11 @@ export const humanLoopApi = {
|
||||
);
|
||||
return { available, mine, all: [], reviewed };
|
||||
},
|
||||
getItem: (id: string) =>
|
||||
getItem: (id: string, options: { signal?: AbortSignal } = {}) =>
|
||||
apiRequest<{
|
||||
item: HumanItem;
|
||||
acceptanceReviews?: { taskId: string; revision: number; decision: string; comment?: string; createdAt: string }[];
|
||||
}>(`/api/human-items/${encodeURIComponent(id)}`),
|
||||
}>(`/api/human-items/${encodeURIComponent(id)}`, options),
|
||||
subjectContentURL: (id: string, side: "left" | "right" = "left") =>
|
||||
`/api/human-items/${encodeURIComponent(id)}/subject-content${side === "right" ? "?side=right" : ""}`,
|
||||
evidenceContentURL: (id: string, artifactId: string) =>
|
||||
|
||||
@ -2266,6 +2266,15 @@ export interface HumanItem {
|
||||
display?: HumanTaskDisplayContext;
|
||||
}
|
||||
|
||||
/** Explicit list-only identity. A workbench must load HumanItem detail first. */
|
||||
export type HumanPoolSummary = Omit<HumanItem, "subject"> & {
|
||||
kind: "pool-summary";
|
||||
durationSeconds?: number;
|
||||
subject: Omit<HumanItem["subject"], "payload">;
|
||||
};
|
||||
export const isHumanPoolSummary = (item: HumanItem | HumanPoolSummary): item is HumanPoolSummary =>
|
||||
"kind" in item && item.kind === "pool-summary";
|
||||
|
||||
export interface HumanReject {
|
||||
id: string;
|
||||
organizationId: string;
|
||||
|
||||
@ -39,7 +39,7 @@ try {
|
||||
return route.fulfill({ status: 409, json: { error: "任务已被其他人领取。" } });
|
||||
}
|
||||
if (!["GET", "HEAD"].includes(r.method())) report.writes.push(u.pathname);
|
||||
if (u.pathname === "/api/human-items") {
|
||||
if (["/api/human-items", "/api/human-item-summaries"].includes(u.pathname)) {
|
||||
report.requests.push(Object.fromEntries(u.searchParams));
|
||||
if (u.searchParams.get("pool") === "available" && u.searchParams.get("search") === "FYLX_SEARCH_0612")
|
||||
return route.fulfill({ status: 504, json: { error: "合成可领取池超时" } });
|
||||
@ -63,8 +63,13 @@ try {
|
||||
cards = page.getByTestId("annotation-task-card"),
|
||||
more = page.getByTestId("task-pool-show-more");
|
||||
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 200);
|
||||
assert.match(await available.innerText(), /已加载 200\+/);
|
||||
report.cases.push("bounded initial200 and explicitly loaded count");
|
||||
await page.waitForFunction(() =>
|
||||
document.querySelector('[data-testid="human-task-pool-available"]').textContent.includes("612"),
|
||||
);
|
||||
assert.match(await available.innerText(), /612/);
|
||||
assert.match(await page.getByTestId("human-task-pool-mine").innerText(), /1/);
|
||||
assert.match(await page.getByTestId("human-task-pool-reviewed").innerText(), /1/);
|
||||
report.cases.push("bounded initial200 with independent totals for unvisited tabs");
|
||||
await more.click();
|
||||
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 400);
|
||||
report.cases.push("available cursor loads next200");
|
||||
|
||||
@ -90,7 +90,10 @@ try {
|
||||
continuationWrites.push(route.request().postDataJSON()?.action);
|
||||
if (u.pathname.endsWith("/review-session") && route.request().method() === "POST")
|
||||
sessionWrites.push(route.request().postDataJSON()?.action);
|
||||
if (blockPoolReads && u.pathname === "/api/human-items") {
|
||||
if (
|
||||
blockPoolReads &&
|
||||
["/api/human-items", "/api/human-item-summaries", "/api/human-item-counts"].includes(u.pathname)
|
||||
) {
|
||||
unexpectedPoolReads.push(u.search);
|
||||
return route.fulfill({ status: 503, json: { error: "Synthetic pool outage during layout change" } });
|
||||
}
|
||||
|
||||
@ -348,3 +348,28 @@ test("my-tasks count marks itself partial while another sidebar page exists", as
|
||||
assert.match(tabs, /blindAnnotator && value === "mine" && minePoolCursorRef\.current/);
|
||||
assert.match(tabs, /\? "\+"\s*:\s*""/);
|
||||
});
|
||||
|
||||
test("summary protocol preserves identity and falls back only on missing routes", async () => {
|
||||
const { loadHumanPoolList } = await import("../../src/lib/annotation-project-loading.js");
|
||||
let legacy = 0;
|
||||
const item = { kind: "pool-summary", id: "i", version: 7, canClaim: false, subject: { itemKey: "part" } };
|
||||
const human = {
|
||||
listSummariesPage: async () => ({ summaries: [item], filtersApplied: true }),
|
||||
listItemsPage: async () => {
|
||||
legacy++;
|
||||
return { items: [] };
|
||||
},
|
||||
};
|
||||
assert.equal((await loadHumanPoolList(human, {}, {})).items[0], item);
|
||||
assert.equal(legacy, 0);
|
||||
human.listSummariesPage = async () => {
|
||||
throw Object.assign(new Error("missing"), { status: 404 });
|
||||
};
|
||||
await loadHumanPoolList(human, {}, {});
|
||||
assert.equal(legacy, 1);
|
||||
human.listSummariesPage = async () => {
|
||||
throw Object.assign(new Error("denied"), { status: 403 });
|
||||
};
|
||||
await assert.rejects(loadHumanPoolList(human, {}, {}), /denied/);
|
||||
assert.equal(legacy, 1);
|
||||
});
|
||||
|
||||
22
tests/frontend/review-item-selection.test.mjs
Normal file
22
tests/frontend/review-item-selection.test.mjs
Normal file
@ -0,0 +1,22 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { reviewItemForSelection } from "../../src/lib/review-item-selection.js";
|
||||
|
||||
test("aggregate summaries wait for matching full detail before editing", () => {
|
||||
const summary = { kind: "pool-summary", id: "aggregate", version: 3 };
|
||||
const unrelated = { id: "annotation", version: 9, subject: { payload: { draft: "preserved" } } };
|
||||
assert.equal(reviewItemForSelection(summary, undefined, unrelated), undefined);
|
||||
assert.equal(reviewItemForSelection(summary, unrelated, unrelated), undefined);
|
||||
assert.equal(reviewItemForSelection(summary, { id: "aggregate", version: 2 }, unrelated), undefined);
|
||||
assert.equal(reviewItemForSelection(summary, summary, unrelated), undefined);
|
||||
const detail = { id: "aggregate", version: 3, subject: { payload: { items: [{ id: "source" }] } } };
|
||||
assert.equal(reviewItemForSelection(summary, detail, unrelated), detail);
|
||||
assert.equal(detail.subject.payload.items[0].id, "source");
|
||||
});
|
||||
|
||||
test("legacy package and annotation selections keep their existing detail path", () => {
|
||||
const legacy = { id: "legacy-package-task", version: 2 };
|
||||
assert.equal(reviewItemForSelection(undefined, undefined, legacy), legacy);
|
||||
const full = { id: "aggregate", version: 4, subject: { payload: { items: [] } } };
|
||||
assert.equal(reviewItemForSelection(full, undefined, legacy), full);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user