diff --git a/.scratch/mongo-cpu/spec.md b/.scratch/mongo-cpu/spec.md new file mode 100644 index 00000000..9ef7a19e --- /dev/null +++ b/.scratch/mongo-cpu/spec.md @@ -0,0 +1,32 @@ +# Independent query store for fixed-source Mongo CPU + +Status: claimed +Type: task +Base: 8b33f38e +Branch: codex/mongo-cpu + +## Approved outcome + +Existing Mongo remains authority for annotation, slicing, submission, quality review, final acceptance, permissions and version checks. An independent query Mongo owns thin query facts, projection progress, background aggregation and shared results; Valkey misses read that database. Move counts, dashboards and work records first, then task-pool lists. Production stays unchanged until separate approvals and verified evidence. + +## Capability evidence + +2026-11-12: bounded metadata-only probe against Fanyu's existing API Mongo configuration through an existing pod memory relay: maxWireVersion 25, sessions available, no exposed replica-set/router capability; opening the empty-filter change stream returns classified unsupported-topology (40573). No files were installed in cluster and no customer record was read or written. Sanitized report: local-data/diagnostics/mongo-cpu/cdc-capability-relay.json. Do not infer physical database topology beyond this entry point. Direct host connectivity failed separately and is not CDC evidence. + +## Implementation contract + +- Source adapter atomically tags query-relevant writes in the same document. Default off. Meaningless draft/log/version-only changes do not require repeated projection cycles. Existing filters, business versions and ownership checks remain intact. +- No cross-cluster or source multi-document transaction assumption. Hard deletion uses durable per-operation intents and cancellable per-record claims; recovery must not guess from timestamps or restore deleted/old annotation snapshots. +- Source dirty reads use small explicit indexes and bounded batches. Each projection is durable/idempotent before source marker acknowledgment; acknowledgment is conditional on the exact source revision. Restart resumes a persisted one-time bootstrap checkpoint, then polls only dirty records. No full source scan per browser query or periodic report build. +- The read database receives allowlisted metadata/facts only, excluding annotations, descriptions, opinion text, media, passwords, tokens and signed links. Preserve legacy fact interpretation and exact statistics reducers. +- Query read selection applies only to nominated GET paths. Authentication, current authorization, source task detail, command validation, claim CAS and workflow workers remain on authority. No stale permission grants; use authoritative page checks and pending visibility exclusions where needed. +- Target health/coverage/version are checked before serving. Cache failure reads the query store; target failure cannot cause a full-source fallback storm. Existing tasks remain operable through authority. Old paths stay available behind an explicit rollback switch. +- Count/list matching, pagination and server-shipped canClaim preserve existing contracts. Counts use a stated projection watermark; actual claim/decision is authoritative. Missing/incomplete data must not become zero or silently drop tasks. + +## Validation + +Real isolated source standalone + separate query replica set; atomic tagging, no-op suppression, deletes, delete/submit concurrency, old jobs, duplicate/out-of-order processing, restart, bootstrap races, cache/query-store loss, permission revocation and legacy data. Reuse original reducers and original read path for exhaustive result comparisons. Measure original source CPU/commands/bytes/scans including projection and acknowledgment load, plus query latency and new services' resources. Test 81 then 200/500 actors and larger history; label actual resource constraints, do not turn an uncapped native run into a 4-core capacity promise. Formal release gates follow integration to synchronized dev, never from this unmerged worktree. + +## State + +Local implementation complete on codex/mongo-cpu; default off. CDC verified read-only. Thin report/history/count reads, Valkey and subsequently pool candidates are connected to the independent query database. All authority mutations and permissions/CAS stay on source. New Mongo/Valkey/race/contracts pass; original work-availability and browser regressions pass. Bounded count-path comparisons show source offload, but full HTTP/mixed workload, write amplification, long steady-state and independent resource capacity remain unaccepted. Formal task check retains failures/skips/mapping gaps and a scheduler-concurrency anomaly is recorded, not silently dismissed. See verification.md. No source/cluster indexes or customer data changed; no push, PR, merge or release authorized/performed. diff --git a/.scratch/mongo-cpu/verification.md b/.scratch/mongo-cpu/verification.md new file mode 100644 index 00000000..52f2daf6 --- /dev/null +++ b/.scratch/mongo-cpu/verification.md @@ -0,0 +1,77 @@ +# mongo-cpu verification — 2026-11-12 + +Status: claimed +Type: task +Base: 8b33f38e59d20ea13855c75b68973a78cf74de20 +Branch: codex/mongo-cpu +Scope: isolated synthetic data only; production metadata-only CDC probe. +Production acceptance: NOT complete. All production changes remain unauthorized/unperformed. + +## Passed + +- Freshness: `local-data/diagnostics/mongo-cpu/freshness-final.log`; contains origin/dev and live commit. +- CDC: `cdc-capability-relay.json` in the same directory. Current entry returns unsupported topology40573, sessions available, no exposed set/router. No customer content copied. +- Final dedicated module: `final-replica-gate-6.log`, raw `local-data/query-replica-local/logs/check-c139e638e31c49b2a16aa92bc039aaea.log`. Real source standalone/query RS/Valkey, race detector; no missing, skipped or changed-source cases. Includes old writer repair, revision reset, deletion/lease fences, null/mixed IDs, idempotent receipts, pending filtered/blind counts, source ACL revocation, self-review, configuration fail-closed and healthy independent fact routing. +- Source business contracts with tracking enabled: supplier management/deletion, acceptance authorization/revocation, latest progress; production indexes retained. Adapter supplies missing synthetic source keys on fixture inserts/claims, without changing production methods or assertions. +- Task-level original work availability core/Mongo/advisory browser/recovery: passed. Task browser searches, task transport, slicing/rework, deleted-annotator recovery, dashboard presentation/refresh/stability and original query-serving real components passed. +- Additional exact Mongo cases: `additional-mongo-contracts.log`, all passed. Platform inventory fixture now includes42 real source tasks plus an orphan that must remain excluded; no business gate weakened. Trace, index migration, latest progress and supplier pool conversion passed. +- Final Go baseline: `go-baseline-final-2/summary.json`:68 failures, all70-entry existing baseline, zero new. Original base exported read-only (not a Git worktree) also passed the same baseline gate: `go-base-8b33/summary.json`. Never changed known-failures.json. +- Python compatibility: `python-compatibility-final.log`:37 passed. Four materialization cases still skipped without their Go fixture context, not counted as passed. + +## Original failures retained + +- `verify-task-final.log` / `local-data/test-results/verify-1794504930660420000/report.json`:47 passed,3 incomplete,3 failed, source unchanged. Missing pytest was rerun with existing root venv. Inventory fixture failure was corrected and exact cases passed. Productionstats failures are in the Go known-failure baseline. Optional external coverage/mapping gaps are not declared green. +- `verify-quick-final.log`: sandboxed Node lifecycle check timed out; identical selected frontend command passed in the elevated task gate. No claim that the initial quick run passed. +- Dedicated attempts3/4: older generic fixtures lacked unique source keys. Production unique indexes were retained; synthetic fixture adapter corrected identity on both insert and claim. Attempt5: best-effort shared-cache hit assertion failed once under concurrent verification; final complete run passed unchanged. Cache outage correctness passed throughout; keep transient hit failure for sustained-load follow-up. +- `go-baseline-final/summary.json`: one additional Worker concurrent dispatch failure. `worker-concurrency-repeat.log` reproduced it with memory fixtures; base narrow20/200 runs passed, but the unchanged base under race detection reproduced the same failure (`worker-concurrency-base-race.log`). Full final gate passed. This verifies the anomaly also exists at the task baseline; no scheduler code or baseline was changed. Its root cause remains uncorrected and must be tracked before production acceptance. + +## Bounded performance only + +Dataset26218 synthetic tasks; same existing read indexes both sides, source pending checks and marker/ack traffic included. No customer data. Native runs had no CPU/RSS quotas. Docker has source4CPU/4GiB and query4CPU/8GiB declared limits but the entire VM has only4CPU/~6GiB, so it is not an independent8CPU/12GiB environment. + +Fixed-resource raw artifact: `local-data/diagnostics/mongo-cpu/scale/20261112T172715.078381000Z/report.json` (`complete:false`). + +| All-project actors | Authority p95 ms / errors | New path p95 ms / errors | Source CPU-seconds before / after | +| ------------------ | ------------------------- | ------------------------ | --------------------------------- | +| 81 | 6462 /45 | 259 /0 | 22.568 /0.046 | +| 200 | 21465 /83 | 348 /0 | 74.893 /0.066 | +| 500 | 64638 /247 | 547 /0 | 222.933 /0.197 | + +These are storage-count visitors, not full HTTP or complete workflows. Original baseline failures make the combined comparison test FAIL; never mark it as accepted. Optimized counts match the complete fixture. Old timeout-truncated replies cannot support a strict end-to-end reduction percentage. Source CPU/bytes do decrease in this measured path; no production total CPU/EIP claim. + +## Remaining / next gate + +1. Resolve/reproduce the Worker concurrency anomaly and remaining external task mapping/fixture gaps; keep failed logs. +2. Dedicated independent Linux source4CPU + separately provisioned query/worker/cache resources, exact version/fixtures/index/compression/request script. Full HTTP role/permission parity, cold and warm counts, all-project search/list, dashboards and work records. + 3.81/200/500 actors, three long steady-state rounds, writes/submission and incremental/delete/reconciliation cost included. Record source and target CPU/RSS/IO/scans/bytes, API latency/errors, lag, cache loss, backlog and restart. New500-actor count p95547ms still exceeds the proposed warm500ms goal in this constrained short test. +3. Inventory production writers and index sizes before any source tagging/copy.47 potential new source indexes need a specific cost/rollback plan and explicit authorization. No indexes created in company Mongo. +4. Only after result/permission correctness, source total load reduction and speed targets pass: integrate to dev, synchronize with approval, then prepare a separately authorized release through release.py. Do not release the worktree branch or treat Pod readiness as acceptance. + +## Final query configuration/recovery follow-up + +- Source database and QUERY_SERVING_NAMESPACE are pinned in target control metadata; a different logical source/namespace cannot reuse checkpoints or take over existing facts. This does not claim physical identity across clones that deliberately reuse both settings; topology verification and disaster-recovery planning remain required. +- A cold, temporarily disconnected query client is retained. A cancelable single verifier retries at most once per second, so startup degradation self-recovers without replacing the API reader or scanning authority history. Initialization/readiness failure never becomes a zero report or a core-work gate. +- Full dedicated final suite: `final-replica-complete.log`, raw `local-data/query-replica-local/logs/check-26b729daec844d87afd55c4b9ff5cfcc.log`; no missing/skipped/changed inputs, race detector enabled. Includes real gated-network failure followed by successful automatic recovery, correct environment routing and wrong-authority/namespace rejection. +- Cross-module integration: `verify-integration-final.log`, `local-data/test-results/verify-1794506801679137000/report.json`:47 passed,2 incomplete,1 failed, inputs unchanged. All six integration checks passed (web lint, full format, frontend baseline, Go baseline, build, production artifact boot). Overall remains failed because raw productionstats tests have known failures and optional Mongo/worker cases/mapping gaps remain incomplete. Binding/cold-start additions followed this run and have separate final dedicated evidence; do not present the earlier whole-repo fingerprint as final-code release approval. +- `isolated-capacity-plan.md` and sanitized cluster node/PV inventories are in the ignored diagnostics folder. No suitable unused database PV is established. Storage choice is pending; no company resources or production indexes/data were changed. + +## Final metadata-stampede fix and repeated comparison + +- Final-code attempt `scale/20261112T184107.767161000Z/report.json` exposed70 new-path one-project500 failures with generic not-ready errors. It remains retained, not replaced or accepted. The per-request200ms readiness probe was too fragile during the burst. +- Fixed by sharing only an in-flight readiness fetch via singleflight, with a bounded independent producer and per-caller cancellation. No readiness TTL, data truncation, permission cache or source-history fallback added.100 simultaneous waiters execute exactly one real Mongo find. Full module/race/contracts passed: `final-replica-watermark.log`, raw `local-data/query-replica-local/logs/check-a5cd98306e064edfb491fbc23394459e.log`. +- Same dataset/resources/indexes/script after fix: `scale-watermark-final.log`, immutable `scale/20261112T184936.001112000Z/report.json` and raw `local-data/query-replica-local/logs/check-da9d072b74ed4ea8aad7c1b4cb13f66c.log`. Source fingerprint unchanged during run. All six new-path phases have zero errors; fixture counts exact. Combined test still FAILS because old-path phases time out. This is not production acceptance. + +| Scope/actors | Old p95 ms / errors | Final new p95 ms / errors | Source CPU-seconds old / new | Source reply bytes old / new | +| ------------ | ------------------- | ------------------------- | ---------------------------- | ---------------------------- | +| Single81 | 1294 /0 | 107 /0 | 4.8454 /0.0269 | 44185088 /37012 | +| Single200 | 2889 /0 | 173 /0 | 10.6156 /0.0548 | 109099010 /84018 | +| Single500 | 6644 /57 | 330 /0 | 24.3211 /0.1377 | 241654452 /202518 | +| All81 | 7039 /39 | 203 /0 | 25.1073 /0.0288 | 554784076 /37012 | +| All200 | 21159 /83 | 328 /0 | 74.6761 /0.0693 | 1524127176 /84018 | +| All500 | 63608 /187 | 528 /0 | 221.024 /0.1497 | 4087040050 /202518 | + +Single-project81 source commands243→268: small indexed checks can increase command count while dramatically reducing read volume/CPU. Do not claim every metric decreases. Go client/native host and shared4CPU/~6GiB Docker VM constraints remain; no HTTP/auth/full workflow or source total steady-state CPU claim. Final all-project500 p95528ms still exceeds warm500ms target. + +Final Go gate after watermark coalescing: `go-watermark-final/summary.json`,68 existing failures,0 new. Full formatting is rerun with all new task files staged (`format-staged-final.log`), so untracked-file omission cannot hide formatting failures. Source freshness before delivery: `freshness-delivery.log`, includes origin/dev and live. + +The dedicated replica Docker lab is stopped after verification with named volumes preserved; `python3 scripts/test_query_replica.py --auto` restarts it. Existing27017/27018/27019 services and protected worktrees are not cleaned/reset by this task. No push, PR, dev merge or deployment was performed. diff --git a/.scratch/mongo-cpu/work.md b/.scratch/mongo-cpu/work.md new file mode 100644 index 00000000..1b6d2e03 --- /dev/null +++ b/.scratch/mongo-cpu/work.md @@ -0,0 +1,20 @@ +# mongo-cpu worktree + +Status: claimed +Type: task +Owner: Codex +Branch: codex/mongo-cpu +Base: 8b33f38e59d20ea13855c75b68973a78cf74de20 (fresh dev; contains origin/dev and live) +Worktree: local-data/worktrees/mongo-cpu + +用户明确授权独立worktree mongo-cpu。先只读核实公司Mongo CDC;再实现独立查询Mongo、可靠增量同步,优先计数/看板/作业记录,随后任务池列表。复用既有Valkey与Typesense可选模块。源Mongo仍掌管标注、切片、送检、质检、最终验收、权限与版本;不改客户记录。 + +文件域:查询存储适配、CDC/投影后台、统计与计数/列表读路径、配置及本地隔离验证、运维文档。保护dev原有两项admin测试和旧发布笔记。 + +回归与减负验证通过后在本分支本地提交,报告证据与未验证范围;回dev集成、推送、PR、公司部署及生产数据/索引写入按后续明确授权执行。只读CDC探测不写客户数据、不开新生产服务。 + +先前原库物化版本两轮持续对照通过,第三轮因转入独立读库实施而中止,不作为完整验收。 + +## Current evidence + +Local read-replica implementation and scoped regressions complete; capacity verification remains pending. Keep this worktree and isolated source/query lab for follow-up. Source and query feature switches stay OFF in production. Merge-back only after remaining acceptance and explicit integration request; never release this unmerged branch directly. See [verification](verification.md) for passing, failed and pending evidence. diff --git a/Dockerfile.query-read-builder b/Dockerfile.query-read-builder index 3888692f..96cbd53c 100644 --- a/Dockerfile.query-read-builder +++ b/Dockerfile.query-read-builder @@ -5,10 +5,12 @@ COPY go.mod go.sum ./ RUN GOPROXY="${GOPROXY}" go mod download COPY cmd/query-read-builder ./cmd/query-read-builder COPY internal ./internal +RUN mkdir -p /out/tmp && chmod 1777 /out/tmp RUN CGO_ENABLED=0 go build -trimpath -o /out/query-read-builder ./cmd/query-read-builder # A static binary only needs trusted CA roots; no media/worker tools or app code. FROM scratch +COPY --from=build --chmod=1777 /out/tmp /tmp COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=build /out/query-read-builder /query-read-builder ARG DROBOTICFLOW_BUILD_COMMIT=unknown diff --git a/Dockerfile.query-replica b/Dockerfile.query-replica new file mode 100644 index 00000000..def7d320 --- /dev/null +++ b/Dockerfile.query-replica @@ -0,0 +1,19 @@ +FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS build +WORKDIR /src +ARG GOPROXY=https://proxy.golang.org,direct +COPY go.mod go.sum ./ +RUN GOPROXY="${GOPROXY}" go mod download +COPY cmd/query-replica ./cmd/query-replica +COPY internal ./internal +RUN mkdir -p /out/tmp && chmod 1777 /out/tmp +RUN CGO_ENABLED=0 go build -trimpath -o /out/query-replica ./cmd/query-replica + +# A static binary only needs trusted CA roots; no media/worker tools or app code. +FROM scratch +COPY --from=build --chmod=1777 /out/tmp /tmp +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=build /out/query-replica /query-replica +ARG DROBOTICFLOW_BUILD_COMMIT=unknown +LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} +USER 10001:10001 +ENTRYPOINT ["/query-replica"] diff --git a/cmd/api/main.go b/cmd/api/main.go index b80010b1..d9995d08 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -173,7 +173,14 @@ func main() { }() dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) + dataStore.EnableQuerySourceTracking(os.Getenv("QUERY_SOURCE_TRACKING") == "1") if os.Getenv("ANNOTATION_PROCESSOR_MODE") != "processor" { + closeReplica, replicaErr := dataStore.ConfigureQueryReplicaFromEnvironment(ctx) + if replicaErr != nil { + fmt.Println("[api] independent reporting unavailable; existing work remains enabled") + } else { + defer closeReplica() + } closeServing, servingErr := dataStore.ConfigureQueryServingFromEnvironment() if servingErr != nil { fmt.Println("[api] query accelerator configuration unavailable; using existing reads") diff --git a/cmd/drosync-coordinator/main.go b/cmd/drosync-coordinator/main.go index f5c877fb..5471fcd6 100644 --- a/cmd/drosync-coordinator/main.go +++ b/cmd/drosync-coordinator/main.go @@ -49,6 +49,7 @@ func main() { } defer client.Disconnect(context.Background()) dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) + dataStore.EnableQuerySourceTracking(os.Getenv("QUERY_SOURCE_TRACKING") == "1") if err := bootstrapCoordinator(ctx, dataStore, os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")); err != nil { fmt.Printf("[drosync-coordinator] database bootstrap: %v\n", err) os.Exit(1) diff --git a/cmd/pool-search-backfill/main.go b/cmd/pool-search-backfill/main.go index d251fc73..ee18a5a9 100644 --- a/cmd/pool-search-backfill/main.go +++ b/cmd/pool-search-backfill/main.go @@ -35,6 +35,7 @@ func main() { } defer client.Disconnect(context.Background()) ds := store.NewMongoStore(client.Database(*database)) + ds.EnableQuerySourceTracking(os.Getenv("QUERY_SOURCE_TRACKING") == "1") var result store.PoolSearchBackfillResult if *collection == "humanItems" { result, err = ds.BackfillHumanPoolSearchMetadata(ctx, *org, *apply) diff --git a/cmd/query-cdc-probe/main.go b/cmd/query-cdc-probe/main.go new file mode 100644 index 00000000..47ee249c --- /dev/null +++ b/cmd/query-cdc-probe/main.go @@ -0,0 +1,148 @@ +// query-cdc-probe performs bounded metadata-only CDC capability checks. +// It never writes records or returns customer fields, addresses or resume tokens. +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "flag" + "os" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type result struct { + At time.Time `json:"at"` + ReplicaSet bool `json:"replicaSet"` + Router bool `json:"router"` + MaxWireVersion int `json:"maxWireVersion"` + Sessions bool `json:"sessions"` + Open bool `json:"streamOpen"` + GetMore bool `json:"streamGetMore"` + Resume bool `json:"streamResume"` + Token bool `json:"resumeTokenObserved"` + Failure string `json:"failure,omitempty"` +} + +func classification(err error) string { + if err == nil { + return "" + } + if errors.Is(err, context.DeadlineExceeded) { + return "deadline" + } + var command mongo.CommandError + if errors.As(err, &command) { + switch command.Code { + case 13, 18: + return "authorization" + case 40573: + return "unsupported-topology" + case 286, 280: + return "resume-unavailable" + } + return "mongo-command" + } + return "connection-or-driver" +} +func closeStream(stream *mongo.ChangeStream) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = stream.Close(ctx) +} + +func probe(ctx context.Context, db *mongo.Database) result { + out := result{At: time.Now().UTC()} + var hello struct { + SetName string `bson:"setName"` + Msg string `bson:"msg"` + MaxWire int `bson:"maxWireVersion"` + Session *int `bson:"logicalSessionTimeoutMinutes"` + } + if err := db.Client().Database("admin").RunCommand(ctx, bson.D{{Key: "hello", Value: 1}}).Decode(&hello); err != nil { + out.Failure = classification(err) + return out + } + out.ReplicaSet = hello.SetName != "" + out.Router = hello.Msg == "isdbgrid" + out.MaxWireVersion = hello.MaxWire + out.Sessions = hello.Session != nil + // Impossible operation type: capability check without exporting events. + pipeline := mongo.Pipeline{{{Key: "$match", Value: bson.M{"operationType": "__metadata_only_cdc_probe__"}}}, {{Key: "$project", Value: bson.M{"_id": 1}}}} + stream, err := db.Watch(ctx, pipeline, options.ChangeStream().SetMaxAwaitTime(250*time.Millisecond).SetBatchSize(1)) + if err != nil { + out.Failure = classification(err) + return out + } + out.Open = true + stream.TryNext(ctx) + if err = stream.Err(); err != nil { + closeStream(stream) + out.Failure = classification(err) + return out + } + out.GetMore = true + token := append(bson.Raw(nil), stream.ResumeToken()...) + out.Token = len(token) > 0 + closeCtx, cancel := context.WithTimeout(context.Background(), time.Second) + _ = stream.Close(closeCtx) + cancel() + if !out.Token { + out.Failure = "resume-token-unavailable" + return out + } + resumed, err := db.Watch(ctx, pipeline, options.ChangeStream().SetResumeAfter(token).SetMaxAwaitTime(250*time.Millisecond).SetBatchSize(1)) + if err != nil { + out.Failure = classification(err) + return out + } + resumed.TryNext(ctx) + err = resumed.Err() + closeCtx, cancel = context.WithTimeout(context.Background(), time.Second) + _ = resumed.Close(closeCtx) + cancel() + if err != nil { + out.Failure = classification(err) + return out + } + out.Resume = true + return out +} +func main() { + allow := flag.Bool("production-readonly", false, "allow metadata-only checks using explicitly supplied company credentials") + flag.Parse() + uri, name := os.Getenv("MONGO_URI"), os.Getenv("MONGO_DB") + out := result{At: time.Now().UTC()} + if !*allow || uri == "" || name == "" { + out.Failure = "explicit-readonly-configuration-required" + _ = json.NewEncoder(os.Stdout).Encode(out) + os.Exit(2) + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + opts := options.Client().ApplyURI(uri).SetDirect(true).SetServerSelectionTimeout(5 * time.Second).SetConnectTimeout(4 * time.Second).SetMaxPoolSize(2) + if host := os.Getenv("MONGO_TLS_SERVER_NAME"); host != "" { + if opts.TLSConfig == nil { + opts.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + opts.TLSConfig.ServerName = host + } + client, err := mongo.Connect(ctx, opts) + if err != nil { + out.Failure = classification(err) + } else { + out = probe(ctx, client.Database(name)) + closeCtx, stop := context.WithTimeout(context.Background(), time.Second) + _ = client.Disconnect(closeCtx) + stop() + } + _ = json.NewEncoder(os.Stdout).Encode(out) + if !out.Resume { + os.Exit(1) + } +} diff --git a/cmd/query-cdc-probe/main_test.go b/cmd/query-cdc-probe/main_test.go new file mode 100644 index 00000000..923b9376 --- /dev/null +++ b/cmd/query-cdc-probe/main_test.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + "errors" + "go.mongodb.org/mongo-driver/mongo" + "testing" +) + +func TestClassificationNeverExportsConnectionOrCustomerText(t *testing.T) { + for _, c := range []struct { + e error + want string + }{{context.DeadlineExceeded, "deadline"}, {mongo.CommandError{Code: 40573, Message: "private-address secret"}, "unsupported-topology"}, {mongo.CommandError{Code: 13, Message: "secret"}, "authorization"}, {errors.New("mongodb://private-password@private-address/customer-body"), "connection-or-driver"}} { + if got := classification(c.e); got != c.want { + t.Fatal(got, c.want) + } + } +} diff --git a/cmd/query-read-builder/main.go b/cmd/query-read-builder/main.go index 673e626f..e5f37673 100644 --- a/cmd/query-read-builder/main.go +++ b/cmd/query-read-builder/main.go @@ -3,11 +3,12 @@ package main import ( "context" - "droboticflow/internal/config" "droboticflow/internal/store" "flag" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/readpref" "log" "os" "os/signal" @@ -21,19 +22,22 @@ func main() { flag.Parse() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - cfg, err := config.LoadValidated() - if err != nil { - log.Fatal("invalid builder configuration") - } if os.Getenv("QUERY_SERVING_READS") != "1" { log.Fatal("explicit QUERY_SERVING_READS=1 is required") } - client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.MongoURI)) + uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB") + if uri == "" || db == "" { + log.Fatal("independent query database required; authority builds disabled") + } + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())) if err != nil { log.Fatal("builder database unavailable") } defer client.Disconnect(context.Background()) - s := store.NewMongoStore(client.Database(cfg.MongoDB)) + s := store.NewMongoStore(client.Database(db)) + if s.VerifyQueryReplicaTarget(ctx) != nil { + log.Fatal("projected query database identity unavailable") + } closeServing, err := s.ConfigureQueryServingFromEnvironment() if err != nil { log.Fatal("builder accelerator configuration invalid") diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go new file mode 100644 index 00000000..39ccc1e7 --- /dev/null +++ b/cmd/query-replica/main.go @@ -0,0 +1,85 @@ +// query-replica is a rate-limited, independently deployed projector. It never +// claims tasks, repairs annotations, or starts business background workers. +package main + +import ( + "context" + "droboticflow/internal/store" + "flag" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/readpref" + "log" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + bootstrap := flag.Bool("bootstrap", false, "explicitly allow bounded resumable historical copy") + ensure := flag.Bool("ensure-source-indexes", false, "explicit source index writes; requires production authorization") + writers := flag.Bool("writers-covered", false, "attest all source writers use atomic query-source/v1 markers") + once := flag.Bool("once", false, "one bounded pass only") + interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes") + flag.Parse() + if !*writers || *interval < 100*time.Millisecond || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { + log.Fatal("writer coverage, namespace and bounded polling interval required") + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + connect := func(uri, db string, target bool) (*mongo.Client, *store.MongoStore) { + if uri == "" || db == "" { + log.Fatal("explicit source/query connection required") + } + opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second) + if target { + opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()) + } + c, e := mongo.Connect(ctx, opts) + if e != nil { + log.Fatal("database connection unavailable") + } + return c, store.NewMongoStore(c.Database(db)) + } + src, source := connect(os.Getenv("MONGO_URI"), os.Getenv("MONGO_DB"), false) + defer src.Disconnect(context.Background()) + dst, target := connect(os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB"), true) + defer dst.Disconnect(context.Background()) + q, e := store.NewQueryReplica(source, target) + if e != nil { + log.Fatal("invalid projector configuration") + } + if q.VerifyTopology(ctx) != nil { + log.Fatal("source/query databases are not independently usable") + } + if *ensure && q.EnsureSourceIndexes(ctx) != nil { + log.Fatal("source index preparation failed") + } + if q.EnsureTargetIndexes(ctx) != nil { + log.Fatal("query index preparation failed") + } + if q.Acquire(ctx) != nil { + log.Fatal("projector lease unavailable") + } + for ctx.Err() == nil { + ready, e := q.RunOnce(ctx, *bootstrap) + if e != nil { + log.Print("projector paused; checkpoint retained (no customer payload logged)") + os.Exit(1) + } + if *once { + if !ready { + log.Print("projection incomplete; resume required") + os.Exit(2) + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(*interval): + } + } +} diff --git a/cmd/rework-handoff-plan/main.go b/cmd/rework-handoff-plan/main.go index 39ed6b4f..8639e40a 100644 --- a/cmd/rework-handoff-plan/main.go +++ b/cmd/rework-handoff-plan/main.go @@ -36,6 +36,7 @@ func main() { } defer client.Disconnect(context.Background()) s := store.NewMongoStore(client.Database(*database)) + s.EnableQuerySourceTracking(os.Getenv("QUERY_SOURCE_TRACKING") == "1") encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") if !*apply { diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 9903c11a..43e86f16 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -52,6 +52,7 @@ func main() { }() dataStore := store.NewMongoStore(client.Database(cfg.MongoDB)) + dataStore.EnableQuerySourceTracking(os.Getenv("QUERY_SOURCE_TRACKING") == "1") if err := bootstrap.Run(ctx, dataStore, os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")); err != nil { fmt.Printf("[worker] database bootstrap: %v\n", err) os.Exit(1) diff --git a/docs/adr/0014-independent-query-database.md b/docs/adr/0014-independent-query-database.md new file mode 100644 index 00000000..458f8319 --- /dev/null +++ b/docs/adr/0014-independent-query-database.md @@ -0,0 +1,32 @@ +# 0014:独立查询库与可恢复的增量同步 + +状态:已批准设计,本地实施,尚未生产验收。 + +## 决定 + +原 Mongo 保留业务权威、权限、归属、版本和正式记录。计数、生产与人员看板、作业记录及任务池候选逐步迁到独立 Mongo 查询库;Valkey 只缓存查询事实与按范围隔离的派生结果。Typesense 保持可选候选搜索,不接管权限或最终匹配。 + +2026-11-12 对线上现用连接入口的元数据探测确认:maxWireVersion 25,有会话,但没有可用副本集/路由入口,Change Streams 返回 40573。此证据不推断底层物理部署,也不是权限失败。因此不在原库启动 CDC;新查询库使用副本集,供事务发布及现有读表构建器使用。 + +原库写适配器在同一业务写入中添加私有 `_querySync`。只为影响查询的变化置脏;普通草稿、日志和无业务含义的版本变化不持续产生同步确认写入。送检预留状态依赖版本的例外仍标记。独立投影进程先向查询库多数提交,再按精确修订号确认原库标记。没有原库多文档事务或跨库事务前提。 + +硬删除使用持久意图及可取消的记录标记,恢复先阻止延迟删除,再对照物理身份。缺失结果不能从旧快照恢复标注。限速历史复制、断点和低频摘要对账均由后台执行;恢复只修改私有同步标记及派生事实,不改业务版本或客户标注。 + +## 一致性边界 + +- 缓存保存候选事实,不保存授权结论。账号、成员、供应商授权和实际业务命令仍读原库。 +- 计数请求用原库小索引取得待同步任务/批次;任务覆盖到查询结果,批次策略未同步时读权威记录。干净标记对应已多数提交的查询事实,因此不必每个请求从原库读取全部批次。 +- 列表按本页 ID 批量取得当前归属、范围、状态和版本,再运行原服务端可见性与 `canClaim`。删除/变更不能偷偷截断分页,内部占位不会进入响应。 +- 缓存代次与事实变更在同一查询库事务中递增;失效、容量不足或 Valkey 故障只读查询库。查询库不可用不触发原库全量回源。 +- 所有正常写入角色必须先升级并开启标记,再复制和切读。低频对账是旧维护作业的恢复路径,不是未升级写者的实时 CDC 保证。 +- 不支持观察的核心集合 DDL 在标记启用期间拒绝执行;管理其他普通集合不受影响。先关闭查询读取并制定重建方案,才能重命名/删除受跟踪集合。 + +## 兼容与回退 + +保留原归约器、历史引用回退、日期/时区、去重、分页和服务端权限规则。不复制标注正文、草稿、批注正文、媒体、密码或签名链接。作业详情按已授权结果 ID 从原库补充正式意见。无法安全导出的历史名称会阻止新路径验收,不静默丢记录;原路径在开关关闭时仍可用。 + +报告/任务池是分别默认关闭的开关。回退先关闭 API 新读取,再停后台和标记;不回滚或改写标注。恢复查询库备份或完全重建时必须先关闭新读取、清理旧追平证明并重新对账,不把 Pod 就绪当成数据追平。 + +## 验收 + +详见 [操作与验收说明](../operations/query-replica.md)。必须比较原库总 CPU、扫描和读取字节,包含新索引、标记、确认和对账成本。短时本机测试和无 CPU 配额的 native 测试不构成生产容量验收。 diff --git a/docs/operations/query-replica.md b/docs/operations/query-replica.md new file mode 100644 index 00000000..2076bb4f --- /dev/null +++ b/docs/operations/query-replica.md @@ -0,0 +1,109 @@ +# 独立查询库操作与验收 + +状态:本地实现,默认关闭;无生产数据、索引或集群变更。实现决策见 [ADR 0014](../adr/0014-independent-query-database.md)。现有 [查询读表与缓存](query-serving.md) 在此模式下只作用于查询库。 + +## 数据路径 + +```text +业务命令 → 原 Mongo(原权限/CAS + 同文档同步标记) + ↓ 限速增量投影 / 断点历史复制 + 独立查询 Mongo(多数事务、薄事实) + ↓ 查询库自己的 CDC + query-read-builder → 读表 / Valkey + +计数/看板/作业记录 → 查询库或共享结果 +任务列表 → 查询库选候选 → 按 ID 核对原库当前状态 → 原服务端授权 +领取/保存/切片/送检/质检/最终验收 → 原 Mongo(不改读异步副本) +``` + +原库当前入口已用只读探测验证不支持 CDC(40573),并非 API 总资源不足。独立查询库必须是另一 Mongo 部署,不能只改原服务器上的数据库名。投影器核对 hello 身份,拒绝相同进程/副本集;查询端必须支持事务。聚合器还检查投影库私有身份,不接受直接对原库构建。 + +## 默认关闭的配置 + +凭证只通过既有 loader 私有覆盖文件或 Secret 注入,不写入仓库或证据。投影器、API 与聚合器必须使用相同且按环境隔离的 `QUERY_SERVING_NAMESPACE`;查询库控制表绑定来源数据库名与命名空间,拒绝不同来源配置的投影器接管。它是逻辑环境绑定,不替代物理拓扑核验;同名副本/灾备切换仍需专门恢复计划。 + +| 配置 | 角色与含义 | +| --- | --- | +| `QUERY_SOURCE_TRACKING=1` | API、切片处理器、校验器、Worker、协调器及写维护工具;启用同文档标记 | +| `QUERY_READ_MONGO_URI` / `QUERY_READ_MONGO_DB` | 独立查询库;不得与权威部署相同 | +| `QUERY_REPLICA_REPORTS=1` | API:计数/生产与人员看板/作业统计的目标读取;默认关闭 | +| `QUERY_REPLICA_POOLS=1` | API:任务池候选、计数及管理员标注池;报告模式可用后另启用 | +| `QUERY_SERVING_READS=1` / `QUERY_SERVING_NAMESPACE` | 查询库派生结果、不可变快照及缓存,namespace 必须隔离环境 | +| `QUERY_VALKEY_URL` | 可选共享缓存。失联只回到查询库 | +| `QUERY_TYPESENSE_URL` / `QUERY_TYPESENSE_API_KEY` | 可选搜索候选,后续独立验收;本批不要求启用 | + +源校验 advisory/cutoff 等现行业务配置不因查询迁移改变。查询聚合器与 API 保持同一口径配置。聚合器不需要原库凭证,不能运行业务调度器。 + +投影进程:`query-replica --writers-covered --bootstrap --interval=1s`。`--writers-covered` 是完成写者盘点后的操作声明,不会自动升级旧作业。每集合每次最多200条、连接池4,投影/确认可重放;查询库租约代次使旧进程不能继续发布。历史复制是一次性、可恢复的读取;每6小时对每集合启动一轮限速摘要对账,每次只处理一个批次。 + +新代码只保存白名单字段,维持原 `productionstats`、`workforcestats`、名称匹配和历史回退。保留原标注和正式意见;作业记录正文按选中的正式记录 ID 回原库读取。不能安全复制的 URL/名称不作为“成功迁移”忽略,须先处理兼容项,或保持该路径开关关闭。 + +## 原库索引与写入成本 + +不会在 API 启动创建索引。`query-replica --ensure-source-indexes` 是明确的原库写操作,生产必须另列计划并授权: + +- 当前22个白名单集合,各新增 `query_sync_dirty_v1`(`_querySync.dirty,_id`,只索引 dirty=true)及 `query_sync_delete_claim_v1`(稀疏 deleteClaim)。 +- 私有删除意图集合新增 phase/lease、collection/phase/audit 及 expireAt TTL 三个索引。已确认意图保留7天,未完成意图不自动过期。 +- 合计47个索引;即便初始为空,构建仍需扫描集合。先只读盘点各集合体积、已有等价索引和磁盘余量,逐集合限速构建,记录 CPU/IO 和停止阈值。不得把索引收益忽略掉持续维护成本。 +- 标记与业务写同次提交;只有相关变化需要一次小的确认更新。草稿正文、日志等不做持续投影。删除另外增加意图/标记写入,必须计入真实作业对照。 +- 对账只做元数据 CAS,保留业务版本、状态、归属及标注;不得用老快照回写正文。 + +查询库显式运行 `EnsureTargetIndexes`,复用 `PoolSearchReadIndexes` / `PlatformReadIndexes`,另加精简事实关联索引。不能用与原实现不同的同名索引蒙混对照。查询库计数选择小文档索引,不强制扫描原库为大文档设计的覆盖路径。 + +## 权限、缓存与故障 + +计数缓存只保存候选事实,权限仍由每个请求的原服务端谓词计算。查询库事实代次在投影事务中递增,缓存按组织/筛选/代次隔离。内存预算64MiB、最多32份;合并最多16个并发构建,单构建8秒,取消互不影响。超容量改用查询库流式读取,不能截断结果。Valkey 使用压缩 BSON 保留私有权限元数据,单值仍受4MiB上限,TTL两秒。 + +原库待同步标记屏障使用小索引;最多1000条变更,越界显示暂不可用,不扫描全量权威历史。未同步任务覆盖旧候选,未同步批次策略读原库。物理删除尚未确认时不伪造零计数。列表继续批量核对当前任务,服务器计算 `canClaim`,实际领取/提交仍用原版本与权限校验。 + +查询副本追平证明30秒过期;失联不集中回源。有效查询连接在冷启动时保留,每秒最多一次、支持取消的拓扑/绑定核验,恢复后自动继续;不替换运行中的读者指针,也不要求手动重启 API。统计失败不返回伪造零数据;API 核心作业仍使用权威库。缓存丢失只影响响应耗时。旧代码或维护工具漏报的变化可由限速对账恢复,但持续旧写者、数据库浏览器 DDL 或人工直接写入不属于实时保证,切读前必须盘点并升级/暂停。 + +## 本地验证与验收证据 + +功能环境是独立 standalone 源库27119、query-read-rs 查询库27129;只允许专用合成数据库,测试结束清理各自随机库,不使用共享快照或客户记录。专用 native 数据目录位于本 worktree 的 `local-data/query-replica-local/`。native 只有固定 WiredTiger 配置,没有 CPU/RSS 配额,只能作为功能与读取路径证据。 + +本批专项入口: + +```bash +QUERY_REPLICA_LOCAL_TEST=1 go test -race ./internal/store ./internal/api -run '^TestQueryReplica|^TestQuerySync' -count=1 -v +QUERY_REPLICA_LOCAL_TEST=1 QUERY_REPLICA_SCALE_LOCAL=1 go test ./internal/store -run '^TestQueryReplicaScaleLocal$' -count=1 -v +``` + +对照固定26218条合成任务;两侧安装相同既有读取索引。81/200/500并发分别测试单项目与全部项目,记录结果、p95、原库命令/响应字节及原生进程 CPU,包含同步标记确认与实时屏障。这仍是短时读取路径对照;原库回复字节不是公网 EIP 流量。 + +生产准备前仍须完成: + +1. 包含真实规模历史、多供应商、受限权限和冷缓存的完整业务链;开启标记后的保存/送检/恢复与既有基线对照。 +2. 固定源库4CPU的资源对照,查询侧明确 CPU/RSS/PV/IOPS;81→200→500用户逐级长稳态,三轮、冷启动、集中进入、积压、缓存丢失、查询库重启。观察原库 CPU、扫描量、读取字节与写放大,而不只看页面响应。 +3. 普通看板/计数暖态p95≤500ms、任务池包括冷查询p95≤2s;无新增业务错误、结果/权限等价;资源持续恶化须解释和处理。native 无配额不能替代上述容量门槛。 +4. 查询副本集多数提交、跨节点调度、PV备份恢复与审计;单节点本地副本集不能证明生产高可用。 + +项目 `verify:plan/quick/task` 与正式发布门禁另行保留证据;跳过、超时、已有失败和新增失败分开记录。 + +## 发布顺序与回退 + +本地验证达标后才准备具体公司变更计划,不在此文执行: + +1. 只读盘点写者、旧作业、原库索引/负载、查询库私网与 PV;查询库3副本建议每副本4CPU/8GiB,投影与聚合进程各1–2CPU/2–4GiB,Valkey 主备各1CPU/2GiB,仅为起步测试预算。 +2. 按单独批准计划建设源索引、新查询服务;所有源写者先升级同一核验版本并开启标记。旧维护脚本不再绕过适配器。保留原作业入口。 +3. 限速复制、持续增量,影子对照;先报告/计数,再任务池。基础管理员执行概览仍保留现有有界执行摘要/配置读取,不宣称其全部来源已迁走。 +4. 在不同时扩原库/改带宽的条件下比较总资源。原库确实减负、结果和速度达标后才准备正式切换。 +5. 回退先关 `QUERY_REPLICA_POOLS` / `QUERY_REPLICA_REPORTS`,再停后台和 `QUERY_SOURCE_TRACKING`。保留原有读取、标注和正式记录;私有标记可保留。源索引可在评估后单独批准删除,不以回退为名清理客户数据。 + +源码必须先并入、核验并同步 dev,再按 `scripts/release.py` 准备,不能直接发布未合并工作树。推送、PR、集群变更、原库索引和数据写入均需要各自的明确授权。 + +## 本批验证记录(2026-11-12) + +- 原 Mongo 入口 CDC 探测为不支持拓扑(40573);不把连接失败或权限失败当作此结论。 +- 专项真实 Mongo/Valkey、竞态检测与旧合同回归通过;包括启动配置选择独立薄事实、错误配置不回源、筛选/盲标计数叠加待同步任务、删除恢复、失权、原标注保留及替换写入的 SDK 多选项语义。 +- 工作可用性四阶段门禁通过,现有切片/返工/管理员任务池/搜索/看板浏览器及统计稳定性回归通过。Python 兼容检查37通过、4项物化夹具缺失仍跳过,不能记为全通过。 +- 最终 Go 基线门禁68项既有失败、0项新增;初次曾出现 Worker 双实例并发用例额外失败,20次定向运行也曾失败,完整复跑通过,原始基线在竞态检测下也复现同一失败。初次记录不删除,不将该既有调度问题归为已解决;调度并发仍需独立确认。 +- 固定26218条任务的 Docker 短时全部项目计数:81/200/500并发,最终新路径p95为203/328/528ms,均无错误。旧路径对应有39/83/187次超时,因此整项对照测试失败,原始结果保留;不能用旧路径失败后的残缺结果宣称严格等价对照通过。此测试仅是存储计数访问器,不包含完整HTTP授权和视频。 +- 源容器配额4CPU/4GiB,查询容器4CPU/8GiB,但本机Docker虚拟机实际仅4CPU、约6GiB,两个Mongo共享主机;目标8GiB不是可兑现的独立物理预算。短时减少的源库命令/读取字节和CPU只证明该查询路径的减负,不能承诺生产总CPU降幅或公网带宽降幅。 +- 完整任务级检查仍记录有失败、跳过和外部用例映射缺口;后续修正夹具的领取库存、作业追溯、索引迁移和供应商转换已定向通过。持续写入/删除积压、完整HTTP及后台写放大/CPU/RSS/IO、三轮长稳态、重启和高可用恢复尚未完成。 + +详细命令、原始失败及源码身份记录在该工作树的 `.scratch/mongo-cpu/verification.md` 和忽略目录 `local-data/diagnostics/mongo-cpu/`。本批保持所有生产开关与数据不变,不具备生产容量验收或发布资格。 + +本批只读存储盘点:当前注册的 StorageClass 为静态 `csi-bos`,现有80Gi本地卷和200Gi BOS卷均已占用,未做修改。隔离验收仍需独立SSD持久卷,不能将现有BOS挂载视为已满足数据库文件系统要求。MongoDB要求目录fsync等文件系统语义,并推荐WiredTiger使用XFS,参见[官方生产说明](https://www.mongodb.com/docs/manual/administration/production-notes/)。节点allocatable不等于空闲资源;尚未创建Pod、PV、StorageClass或Secret。 + +定稿性能复测中,单项目500并发曾有70次就绪检查失败。已将重复的水位读取合并为仅在途共享(无跨请求TTL、无权限缓存),以真实Mongo命令监控证明100个同时检查只读一次;相同夹具复测新路径所有档位均零错误。旧路径仍有超时,因此组合对照仍标为失败,而非性能验收通过。最终全部项目500并发p95528ms仍高于暖态500ms目标;81/200档通过的只是存储计数访问器,完整HTTP/业务授权及长稳态仍待验收。 diff --git a/docs/operations/query-serving.md b/docs/operations/query-serving.md index 57351b81..42d776ba 100644 --- a/docs/operations/query-serving.md +++ b/docs/operations/query-serving.md @@ -1,6 +1,6 @@ # 后台查询读表与可选加速组件 -状态:本地实现;生产保持关闭。速度与资源结论见同任务证据,未通过的门槛不能据本说明视为通过。 +状态:本地实现;生产保持关闭。新采用的独立查询库、原库无 CDC 时的增量标记及回退边界见 [独立查询库操作说明](query-replica.md),其规定优先于下文旧同库路径。速度与资源结论见同任务证据,未通过的门槛不能据本说明视为通过。 ## 责任与兼容 @@ -15,7 +15,7 @@ Mongo 仍是任务、权限、归属、版本与正式审查记录的权威存 ## 配置与本地验证 -API、后台分别显式设置 `QUERY_SERVING_READS=1` 与唯一 `QUERY_SERVING_NAMESPACE`,默认关闭。可选 `QUERY_VALKEY_URL`、`QUERY_TYPESENSE_URL`、`QUERY_TYPESENSE_API_KEY` 均通过现有私有环境配置提供;凭证不提交。不同集群即便数据库同名也必须使用不同 namespace。缓存连接异步建立,启动不可用不会阻止 API;缓存请求有短时限和退避。 +聚合后台现在必须显式提供独立的 `QUERY_READ_MONGO_URI` / `QUERY_READ_MONGO_DB`,并通过投影库身份检查;不再接受使用原库直接构建。API、后台分别显式设置 `QUERY_SERVING_READS=1` 与唯一 `QUERY_SERVING_NAMESPACE`,默认关闭。可选 `QUERY_VALKEY_URL`、`QUERY_TYPESENSE_URL`、`QUERY_TYPESENSE_API_KEY` 均通过现有私有环境配置提供;凭证不提交。不同集群即便数据库同名也必须使用不同 namespace。缓存连接异步建立,启动不可用不会阻止 API;缓存请求有短时限和退避。 本机合成依赖:`python3 scripts/test_query_serving.py --start`。仅暴露 loopback 27019/16379/18108;随机测试密钥在忽略目录,Mongo 使用独立 query-rs 和独立数据卷。`--down` 只停这一项目,保留卷;不停止共享快照库和其他项目。Mongo 文件句柄上限显式设为65536,避免多轮隔离测试建索引触发默认低限制。 diff --git a/internal/api/admin_annotation_pool.go b/internal/api/admin_annotation_pool.go index b921a2a3..c0b48015 100644 --- a/internal/api/admin_annotation_pool.go +++ b/internal/api/admin_annotation_pool.go @@ -87,6 +87,7 @@ func (s *server) adminAnnotationCounts(w http.ResponseWriter, r *http.Request, q } func (s *server) handleAdminAnnotationPool(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(store.WithQueryReplicaPoolRead(r.Context())) q, ok := s.adminAnnotationPoolQuery(w, r) if !ok { return diff --git a/internal/api/human_list_reads.go b/internal/api/human_list_reads.go index f8cc4296..a0353a1b 100644 --- a/internal/api/human_list_reads.go +++ b/internal/api/human_list_reads.go @@ -44,9 +44,47 @@ func (server *server) withHumanListReads(ctx context.Context, org, project strin if err != nil { return ctx, err } + replicaCounts := false + if reader, ok := server.store.(interface{ UsesQueryReplicaCountRead(context.Context) bool }); ok { + replicaCounts = reader.UsesQueryReplicaCountRead(ctx) + } + if replicaCounts { + for _, id := range ids { + cache.batches[id] = nil + } + } for i := range batches { cache.batches[batches[i].ID] = &batches[i] } + if replicaCounts { + for len(batches) > 0 { + sources := []string{} + for _, b := range batches { + if b.SourceBatchID != "" { + if _, loaded := cache.batches[b.SourceBatchID]; !loaded { + sources = append(sources, b.SourceBatchID) + cache.batches[b.SourceBatchID] = nil + } + } + } + if len(sources) == 0 { + break + } + batches, err = server.store.ListHumanBatchAccessRecords(ctx, org, sources) + if err != nil { + return nil, err + } + for i := range batches { + cache.batches[batches[i].ID] = &batches[i] + } + } + } + } + // Counts never hydrate task details or compute response-only claim flags. + // Keep authorization batch/member caches, but do not retain a second copy + // of every task's history join across the exhaustive count scan. + if reader, ok := server.store.(interface{ UsesQueryReplicaCountRead(context.Context) bool }); ok && reader.UsesQueryReplicaCountRead(ctx) { + return context.WithValue(ctx, humanListReadsKey{}, cache), nil } // A later task join may include linked records absent from the actor's // page. Replace each task's group in that read's order; merging the actor diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 3f9a3157..6c402569 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -410,6 +410,7 @@ func (server *server) prepareHumanPoolRead(writer http.ResponseWriter, request * } func (server *server) readHumanItemsList(writer http.ResponseWriter, request *http.Request) (*humanItemListPage, *http.Request) { + request = request.WithContext(store.WithQueryReplicaPoolRead(request.Context())) finishPrepare := store.ObserveReadPhase(request.Context(), "prepare") pool, query, limit, request, ok := server.prepareHumanPoolRead(writer, request) finishPrepare() @@ -587,6 +588,9 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht // filterHumanPoolItem is shared by metadata selection and final response checks. // It must not inspect Subject.Payload or ProcessingJob.Input. func (server *server) filterHumanPoolItem(request *http.Request, item domain.HumanItemRecord, pool string, query domain.HumanItemQuery, batchByID map[string]*domain.HumanBatchRecord) (domain.HumanItemRecord, bool, error) { + if item.QueryReplicaOmit { + return item, false, nil + } principal, _ := currentPrincipal(request) batch, loaded := batchByID[item.BatchID] if !loaded { @@ -688,6 +692,9 @@ func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain } } if !stable { + if replica, ok := server.store.(interface{ UsesQueryReplicaPool(context.Context) bool }); ok && replica.UsesQueryReplicaPool(request.Context()) { + return nil, store.ErrQueryReplicaNotReady + } current, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query) if err == nil && reviewedAcceptanceCategory(query) { err = server.projectReviewedAcceptanceStatuses(request.Context(), server.requestOrganizationID(request), current, batches) diff --git a/internal/api/human_pool_reads.go b/internal/api/human_pool_reads.go index a8a2c252..d14408c9 100644 --- a/internal/api/human_pool_reads.go +++ b/internal/api/human_pool_reads.go @@ -198,6 +198,7 @@ func (s *server) handleHumanPoolCounts(w http.ResponseWriter, r *http.Request) { // Reuse exactly the list's query preparation and per-item authorization. Only // transport/order and response-only claim flags differ. State is request-local. func (s *server) countHumanPoolCandidates(w http.ResponseWriter, r *http.Request) (int, *http.Request, bool) { + r = r.WithContext(store.WithQueryReplicaCountRead(r.Context())) pool, query, _, r, ok := s.prepareHumanPoolRead(w, r) if !ok { return 0, r, false diff --git a/internal/api/query_replica_local_test.go b/internal/api/query_replica_local_test.go new file mode 100644 index 00000000..c5b2e308 --- /dev/null +++ b/internal/api/query_replica_local_test.go @@ -0,0 +1,198 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func TestQueryReplicaPoolPermissionRevocationAndSelfReviewLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("isolated local source and query replica required") + } + t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory") + ctx := t.Context() + clients := []*mongo.Client{} + stores := []*store.MongoStore{} + for _, uri := range []string{"mongodb://127.0.0.1:27119/?directConnection=true", "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs"} { + c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + if e != nil { + t.Fatal(e) + } + db := c.Database(domain.NewID("query_api_test")) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = db.Drop(ctx) + _ = c.Disconnect(ctx) + }) + clients = append(clients, c) + stores = append(stores, store.NewMongoStore(db)) + } + source, target := stores[0], stores[1] + _ = clients + source.EnableQuerySourceTracking(true) + if e := source.EnsureIndexes(ctx); e != nil { + t.Fatal(e) + } + f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1) + project, e := f.store.GetProject(ctx, f.item.ProjectID) + if e != nil { + t.Fatal(e) + } + if e = source.UpsertProject(ctx, *project); e != nil { + t.Fatal(e) + } + supplier, e := f.store.GetSupplier(ctx, f.quality.SupplierID) + if e != nil { + t.Fatal(e) + } + if e = source.UpsertSupplier(ctx, *supplier); e != nil { + t.Fatal(e) + } + for i, u := range []domain.UserRecord{f.cutter, f.annotator, f.quality} { + u.Phone = fmt.Sprint(13000000001 + i) + if e = source.UpsertUser(ctx, u); e != nil { + t.Fatal(e) + } + } + membership := domain.ProjectAnnotationMembershipRecord{ID: "quality-membership", OrganizationID: f.quality.OrganizationID, ProjectID: f.item.ProjectID, UserID: f.quality.ID, WorkgroupID: f.task.WorkgroupID, VendorID: f.task.VendorID, AssignmentRole: domain.AnnotationRoleQualityReviewer} + if e = source.UpsertProjectAnnotationMembership(ctx, membership); e != nil { + t.Fatal(e) + } + member := domain.ProjectMemberRecord{ID: "project-quality", OrganizationID: f.quality.OrganizationID, ProjectID: f.item.ProjectID, UserID: f.quality.ID, SupplierID: f.quality.SupplierID} + if e = source.UpsertProjectMember(ctx, member); e != nil { + t.Fatal(e) + } + if e = source.UpsertAnnotationTask(ctx, f.task); e != nil { + t.Fatal(e) + } + if e = source.UpsertHumanBatch(ctx, f.batch); e != nil { + t.Fatal(e) + } + f.item.Subject.Payload = map[string]any{"annotationDraft": "latest annotation must remain"} + if e = source.UpsertHumanItem(ctx, f.item); e != nil { + t.Fatal(e) + } + q, e := store.NewQueryReplica(source, target) + if e != nil { + t.Fatal(e) + } + for _, fn := range []func(context.Context) error{q.VerifyTopology, q.EnsureSourceIndexes, q.EnsureTargetIndexes, q.Acquire} { + if e = fn(ctx); e != nil { + t.Fatal(e) + } + } + for n := 0; n < 100; n++ { + ready, e := q.RunOnce(ctx, true) + if e != nil { + t.Fatal(e) + } + if ready { + break + } + if n == 99 { + t.Fatal("projection incomplete") + } + } + if e = source.ConfigureQueryReplicaReports(target, time.Minute); e != nil { + t.Fatal(e) + } + source.EnableQueryReplicaPoolReads(true) + handler := NewHandler(source, nil) + request := func(path string) *httptest.ResponseRecorder { + r := httptest.NewRequest(http.MethodGet, path, nil) + r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: f.quality, Organization: domain.OrganizationRecord{ID: f.quality.OrganizationID, Status: domain.OrganizationStatusActive}})) + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + path := "/api/human-item-summaries?pluginId=annotation-qa&pool=available&stageIds=semantic_review" + result := request(path) + var body struct { + Summaries []humanPoolSummary `json:"summaries"` + } + if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &body) != nil || len(body.Summaries) != 1 || body.Summaries[0].CanClaim == nil || !*body.Summaries[0].CanClaim { + t.Fatalf("initial claim flag %d %s", result.Code, result.Body.String()) + } + // Do not project this change. Eligibility must still use current source data. + f.item.LatestAuthorUserID = f.quality.ID + f.item.Version++ + if e = source.UpsertHumanItem(ctx, f.item); e != nil { + t.Fatal(e) + } + result = request(path) + body.Summaries = nil + if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &body) != nil || len(body.Summaries) != 1 || body.Summaries[0].CanClaim == nil || *body.Summaries[0].CanClaim { + t.Fatalf("self review escaped current authority %d %s", result.Code, result.Body.String()) + } + + // A batch policy change not yet projected must affect this very next count. + f.item.LatestAuthorUserID = f.annotator.ID + f.item.Version++ + if e = source.UpsertHumanItem(ctx, f.item); e != nil { + t.Fatal(e) + } + changed := f.batch + changed.Stages = []domain.HumanStageDefinition{{ID: "different-stage", Kind: domain.HumanStageReview, RequiredRole: string(domain.UserRoleQualityReviewer)}} + if e = source.UpsertHumanBatch(ctx, changed); e != nil { + t.Fatal(e) + } + countPath := "/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review" + result = request(countPath) + var counted struct { + Counts map[string]int `json:"counts"` + } + if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &counted) != nil || counted.Counts["available"] != 0 { + t.Fatalf("pending policy bypassed authority: %d %s", result.Code, result.Body.String()) + } + if e = source.UpsertHumanBatch(ctx, f.batch); e != nil { + t.Fatal(e) + } + for n := 0; n < 100; n++ { + ready, e := q.RunOnce(ctx, true) + if e != nil { + t.Fatal(e) + } + if ready { + break + } + } + result = request(countPath) + if result.Code != 200 || json.Unmarshal(result.Body.Bytes(), &counted) != nil || counted.Counts["available"] != 1 { + t.Fatalf("fact epoch retained stale count: %d %s", result.Code, result.Body.String()) + } + if _, e = source.DeleteProjectMember(ctx, member.ID); e != nil { + t.Fatal(e) + } + for _, p := range []string{path, "/api/human-item-counts?pluginId=annotation-qa&stageIds=semantic_review"} { + result = request(p) + if result.Code != 200 { + t.Fatalf("revocation request failed %d %s", result.Code, result.Body.String()) + } + var response map[string]any + if e = json.Unmarshal(result.Body.Bytes(), &response); e != nil { + t.Fatal(e) + } + if list, ok := response["summaries"].([]any); ok && len(list) != 0 { + t.Fatal("revoked project still visible") + } + if counts, ok := response["counts"].(map[string]any); ok && counts["available"] != float64(0) { + t.Fatal("revoked count cached") + } + } + item, e := source.GetHumanItem(ctx, f.item.ID) + if e != nil || item.Subject.Payload["annotationDraft"] != "latest annotation must remain" { + t.Fatal("read path mutated annotation", e) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index b33c5615..07c79710 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1642,7 +1642,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http. summary.Jobs.Failed = summary.Tasks.Failed } enrichDashboardOperatorInventory(&summary, operators) - server.enrichDashboardHumanWork(request.Context(), request, &summary) + if err := server.enrichDashboardHumanWork(request.Context(), request, &summary); err != nil { + renderError(writer, http.StatusServiceUnavailable, err) + return + } if includeWorkforce { if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -1664,7 +1667,10 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http. return } enrichDashboardOperatorInventory(&summary, operators) - server.enrichDashboardHumanWork(request.Context(), request, &summary) + if err := server.enrichDashboardHumanWork(request.Context(), request, &summary); err != nil { + renderError(writer, http.StatusServiceUnavailable, err) + return + } if includeWorkforce { if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -1682,16 +1688,18 @@ type dashboardHumanWorkCounter interface { DashboardHumanWorkStats(ctx context.Context, organizationID string) (store.DashboardHumanWorkStats, error) } -func (server *server) enrichDashboardHumanWork(ctx context.Context, request *http.Request, summary *domain.DashboardSummary) { +func (server *server) enrichDashboardHumanWork(ctx context.Context, request *http.Request, summary *domain.DashboardSummary) error { if summary == nil { - return + return nil } organizationID := server.requestOrganizationID(request) if counter, ok := server.store.(dashboardHumanWorkCounter); ok { - if stats, err := counter.DashboardHumanWorkStats(ctx, organizationID); err == nil { - applyDashboardHumanWorkStats(summary, stats) + stats, err := counter.DashboardHumanWorkStats(ctx, organizationID) + if err != nil { + return err } - return + applyDashboardHumanWorkStats(summary, stats) + return nil } // Fallback for store implementations without the counting interface: // same legacy semantics, fetched in parallel. @@ -1759,6 +1767,7 @@ func (server *server) enrichDashboardHumanWork(ctx context.Context, request *htt } } summary.Tasks.Waiting = summary.Tasks.PendingAnnotation + summary.Tasks.PendingQuality + summary.Tasks.PendingAcceptance + return nil } // applyDashboardHumanWorkStats mirrors the legacy enrich loop's summary diff --git a/internal/domain/human_loop.go b/internal/domain/human_loop.go index faa03361..5d3b2141 100644 --- a/internal/domain/human_loop.go +++ b/internal/domain/human_loop.go @@ -174,6 +174,7 @@ type AccountTaskHandoff struct { } type HumanItemRecord struct { + QueryReplicaOmit bool `json:"-" bson:"-"` AnnotationEpoch int `json:"-" bson:"annotationEpoch,omitempty"` AccountHandoff *AccountTaskHandoff `json:"-" bson:"accountHandoff,omitempty"` ReworkRoutingPending bool `json:"-" bson:"reworkRoutingPending,omitempty"` diff --git a/internal/domain/types.go b/internal/domain/types.go index a6dac9d8..e1becc38 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -846,6 +846,8 @@ type RunRetryAttempt struct { } type RunRecord struct { + // QueryExceptionCategory is a private read-replica fact, never client input/output. + QueryExceptionCategory string `json:"-" bson:"queryExceptionCategory,omitempty"` RetryHistory []RunRetryAttempt `json:"-" bson:"retryHistory,omitempty"` DeliveryOutcome string `json:"deliveryOutcome,omitempty" bson:"deliveryOutcome,omitempty"` ProcessingProfile string `json:"processingProfile,omitempty" bson:"processingProfile,omitempty"` diff --git a/internal/productionstats/current.go b/internal/productionstats/current.go index e77306c5..1fd1b1ff 100644 --- a/internal/productionstats/current.go +++ b/internal/productionstats/current.go @@ -13,7 +13,13 @@ import ( // exceptionCategory keeps operational failures actionable without changing the // source-level attention total. It is derived from persisted failure text for // compatibility with older runs that do not have a structured failure code. +func ExceptionCategory(r domain.RunRecord) string { return exceptionCategory(r) } func exceptionCategory(r domain.RunRecord) string { + switch r.QueryExceptionCategory { + case "metadata_identity_conflict", "metadata_pending", "source_pts", "metadata_pts_mismatch", "source_changed", "metadata_identity", "legacy_metadata_identity", "other": + return r.QueryExceptionCategory + } + message := r.Error if r.Failure != nil { message = r.Failure.Message + "\n" + message diff --git a/internal/store/acceptance_authorization.go b/internal/store/acceptance_authorization.go index 8c6fae1c..d31b5d08 100644 --- a/internal/store/acceptance_authorization.go +++ b/internal/store/acceptance_authorization.go @@ -56,7 +56,7 @@ func (s *MemoryStore) SetSupplierAcceptanceAuthorization(_ context.Context, org, } func (s *MongoStore) GetSupplierAcceptanceAuthorization(ctx context.Context, org, supplier string) (*domain.SupplierAcceptanceAuthorization, error) { v := emptyAcceptanceAuthorization(org, supplier) - err := s.db.Collection("supplierAcceptanceAuthorizations").FindOne(ctx, bson.M{"_id": v.ID}).Decode(&v) + err := s.queryCollection("supplierAcceptanceAuthorizations").FindOne(ctx, bson.M{"_id": v.ID}).Decode(&v) if errors.Is(err, mongo.ErrNoDocuments) { err = nil } @@ -82,7 +82,7 @@ func (s *MongoStore) SetSupplierAcceptanceAuthorization(ctx context.Context, org v.Version++ v.UpdatedAt = nowISO() v.History = append(v.History, domain.SupplierAcceptanceAuthorizationChange{Enabled: enabled, ActorID: actor, At: v.UpdatedAt}) - c := s.db.Collection("supplierAcceptanceAuthorizations") + c := s.queryCollection("supplierAcceptanceAuthorizations") if version == 0 { _, err = c.InsertOne(ctx, v) } else { @@ -107,7 +107,7 @@ func (s *MongoStore) SetSupplierAcceptanceAuthorization(ctx context.Context, org // a newly-created reviewer must not accidentally become a legacy reviewer. func (s *MongoStore) SnapshotLegacyAcceptanceAccounts(ctx context.Context, org string) error { _, err := s.withAccountMutationLock(ctx, "suppliers:"+normalizedOrganizationID(org), func(ctx context.Context) (any, error) { - c := s.db.Collection("acceptanceCompatibilitySnapshots") + c := s.queryCollection("acceptanceCompatibilitySnapshots") id := normalizedOrganizationID(org) var snap struct { UserIDs []string `bson:"userIds"` diff --git a/internal/store/account_admin_guard.go b/internal/store/account_admin_guard.go index 03b729fe..0f296723 100644 --- a/internal/store/account_admin_guard.go +++ b/internal/store/account_admin_guard.go @@ -40,7 +40,7 @@ func removesActiveAdmin(old, next domain.UserRecord) bool { // MongoDB, so transaction retries recount administrators before committing. // Standalone MongoDB uses the same durable mutation guard without a transaction. func (store *MongoStore) updateAdministrator(ctx context.Context, item domain.UserRecord, expected *domain.UserRecord) error { - guards := store.db.Collection("account_mutation_guards") + guards := store.queryCollection("account_mutation_guards") // Create the shared lock outside the transaction to avoid first-use upsert races. _, err := guards.UpdateOne(ctx, bson.M{"_id": "administrators"}, bson.M{"$setOnInsert": bson.M{"version": 0}}, options.Update().SetUpsert(true)) if err != nil && !mongo.IsDuplicateKeyError(err) { diff --git a/internal/store/account_mutation_lock.go b/internal/store/account_mutation_lock.go index 3b664b8e..d1ff539e 100644 --- a/internal/store/account_mutation_lock.go +++ b/internal/store/account_mutation_lock.go @@ -68,7 +68,7 @@ func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context, log.Printf("[account-mutation-lock] event=%s resource=%s key_hash=%x elapsed_ms=%d", event, strings.SplitN(key, ":", 2)[0], digest[:6], time.Since(startedAt).Milliseconds()) } - col := s.db.Collection("accountMutationLocks") + col := s.queryCollection("accountMutationLocks") _, e := col.InsertOne(ctx, bson.M{"_id": key, "locked": false}) if e != nil && !mongo.IsDuplicateKeyError(e) { return nil, e diff --git a/internal/store/admin_annotation_pool.go b/internal/store/admin_annotation_pool.go index 221a1216..36f93f91 100644 --- a/internal/store/admin_annotation_pool.go +++ b/internal/store/admin_annotation_pool.go @@ -342,6 +342,11 @@ func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q return cur.Err() } func (s *MongoStore) ListAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { + if target, err := s.poolReplica(ctx); err != nil { + return nil, err + } else if target != nil { + return s.replicaAdminPool(ctx, target, org, q) + } ids := []string{} statuses := map[string]string{} err := s.scanAdminAnnotationPool(ctx, org, q, func(i domain.HumanItemRecord) bool { @@ -360,6 +365,11 @@ func (s *MongoStore) ListAdminAnnotationPool(ctx context.Context, org string, q } func (s *MongoStore) CountAdminAnnotationPools(ctx context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) { + if target, err := s.poolReplica(ctx); err != nil { + return AdminAnnotationPoolCounts{}, err + } else if target != nil { + return target.CountAdminAnnotationPools(ctx, org, q) + } if q.Capture == "" && q.NodeID == "" && q.ArtifactID == "" { counts := AdminAnnotationPoolCounts{} err := s.scanAdminPoolMetadata(ctx, org, q, true, func(item domain.HumanItemRecord) bool { diff --git a/internal/store/annotation_claim_read.go b/internal/store/annotation_claim_read.go index 1b39003b..cc799abf 100644 --- a/internal/store/annotation_claim_read.go +++ b/internal/store/annotation_claim_read.go @@ -165,7 +165,7 @@ func (s *MongoStore) EnrichAnnotationSourceStatuses(ctx context.Context, items [ return nil } - cur, err = s.db.Collection("annotationIndexJobs").Find(ctx, bson.M{"id": bson.M{"$in": jobIDs}}, options.Find().SetMaxTime(8*time.Second).SetBatchSize(400).SetProjection(bson.M{"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "runId": 1, "status": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1})) + cur, err = s.queryCollection("annotationIndexJobs").Find(ctx, bson.M{"id": bson.M{"$in": jobIDs}}, options.Find().SetMaxTime(8*time.Second).SetBatchSize(400).SetProjection(bson.M{"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "runId": 1, "status": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1})) if err != nil { return err } diff --git a/internal/store/annotation_operations.go b/internal/store/annotation_operations.go index 75198b88..227a9f5f 100644 --- a/internal/store/annotation_operations.go +++ b/internal/store/annotation_operations.go @@ -554,7 +554,7 @@ func mongoAnnotationFilter(organizationID string, extra bson.M) bson.M { return filter } -func mongoGetByID[T any](ctx context.Context, collection *mongo.Collection, id string) (*T, error) { +func mongoGetByID[T any](ctx context.Context, collection mongoCollection, id string) (*T, error) { var item T if err := collection.FindOne(ctx, bson.M{"id": strings.TrimSpace(id)}).Decode(&item); err != nil { if err == mongo.ErrNoDocuments { @@ -565,7 +565,7 @@ func mongoGetByID[T any](ctx context.Context, collection *mongo.Collection, id s return &item, nil } -func mongoList[T any](ctx context.Context, collection *mongo.Collection, filter bson.M, sortFields bson.D) ([]T, error) { +func mongoList[T any](ctx context.Context, collection mongoCollection, filter bson.M, sortFields bson.D) ([]T, error) { cursor, err := collection.Find(ctx, filter, options.Find().SetSort(sortFields)) if err != nil { return nil, err @@ -573,7 +573,7 @@ func mongoList[T any](ctx context.Context, collection *mongo.Collection, filter return decodeMany[T](ctx, cursor) } -func mongoUpsert(ctx context.Context, collection *mongo.Collection, id string, item any) error { +func mongoUpsert(ctx context.Context, collection mongoCollection, id string, item any) error { return supplierGuardedMongoWrite(ctx, collection, id, item, func(ctx context.Context, d bson.M) error { allow, e := projectAllowsUpsert(ctx, collection.Database(), deletionString(d, "projectId")) if e != nil { diff --git a/internal/store/annotation_output.go b/internal/store/annotation_output.go index 3014d6fd..7e95051b 100644 --- a/internal/store/annotation_output.go +++ b/internal/store/annotation_output.go @@ -129,6 +129,11 @@ func (s *MemoryStore) LoadAnnotationOutput(_ context.Context, org string, projec type outputSourceBudget struct{ bytes, documents int } func (s *MongoStore) LoadAnnotationOutput(ctx context.Context, org string, projects []string) (workforcestats.OutputSnapshot, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return workforcestats.OutputSnapshot{}, err + } else if replica != nil { + return replica.LoadAnnotationOutput(ctx, org, projects) + } return s.loadAnnotationOutput(ctx, org, projects, outputSourceBudget{workforcestats.OutputSourceMaxBytes, workforcestats.OutputSourceMaxDocuments}) } func (s *MongoStore) loadAnnotationOutput(ctx context.Context, org string, projects []string, budget outputSourceBudget) (workforcestats.OutputSnapshot, error) { @@ -154,7 +159,7 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro var firstReadErr error var firstReadFailure sync.Once totalBytes, totalDocs := 0, 0 - read := func(c *mongo.Collection, filter, projection bson.M, out any) (err error) { + read := func(c mongoCollection, filter, projection bson.M, out any) (err error) { select { case sem <- struct{}{}: case <-readCtx.Done(): @@ -206,7 +211,7 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro // Bound indexed related-key queries independently of lifetime history size. // The shared semaphore and cumulative budgets still cover every batch. - readRelated := func(c *mongo.Collection, key string, ids []string, projection bson.M, out any, constraints ...bson.M) error { + readRelated := func(c mongoCollection, key string, ids []string, projection bson.M, out any, constraints ...bson.M) error { unique := make([]string, 0, len(ids)) seen := make(map[string]bool, len(ids)) for _, id := range ids { @@ -251,13 +256,13 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro // (their formal problem facts already come from reworkBaselines). draftNeeded := bson.M{"stageId": bson.M{"$in": workforcestats.OutputAcceptanceStageIDs()}, "status": "completed", "decision": "rejected"} projection := fields("_id", "operationId", "projectId", "userId", "itemId", "taskId", "stageId", "decision", "status", "sourceItemIds", "reviewerSupplierId", "createdAt", "completedAt", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.worklog.id") - if err := read(s.db.Collection("reviewOperations"), bson.M{"$and": bson.A{f, bson.M{"$nor": bson.A{draftNeeded}}}}, projection, &d.Operations); err != nil { + if err := read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, bson.M{"$nor": bson.A{draftNeeded}}}}, projection, &d.Operations); err != nil { return err } for _, name := range []string{"draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId"} { projection[name] = 1 } - return read(s.db.Collection("reviewOperations"), bson.M{"$and": bson.A{f, draftNeeded}}, projection, &d.Operations) + return read(s.queryCollection("reviewOperations"), bson.M{"$and": bson.A{f, draftNeeded}}, projection, &d.Operations) }) g.Go(func() error { return read(s.deliveryPackages, f, fields("id", "itemIds", "status", "manifestDigest", "releasePartitionId", "parentRunId", "mode", "temporary", "qualitySummary.prePackageAcceptance", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.reviewerUserId", "acceptanceReviews.createdAt"), &d.Packages) @@ -285,7 +290,7 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro runIDs = append(runIDs, c.RunID) captureIDs = append(captureIDs, c.ID) } - readDirectory := func(table string, c *mongo.Collection, key string, keys []string, projection bson.M, out any, constraints ...bson.M) error { + readDirectory := func(table string, c mongoCollection, key string, keys []string, projection bson.M, out any, constraints ...bson.M) error { if spool == nil { return readRelated(c, key, keys, projection, out, constraints...) } @@ -358,10 +363,10 @@ func (s *MongoStore) loadAnnotationOutputTo(ctx context.Context, org string, pro return readDirectory("items", s.humanEvents, "itemId", itemIDs, fields("id", "itemId", "stageId", "eventType", "actorUserId", "createdAt", "payload.operationId", "payload.sourceItemIds", "payload.deliveryPackageId", "payload.reviewedRevision"), &d.HumanEvents, bson.M{"stageId": bson.M{"$in": workforcestats.OutputAcceptanceStageIDs()}}) }) g.Go(func() error { - return read(s.db.Collection("reworkBaselines"), f, fields("_id", "itemId", "round", "stageId", "reviewOperationId", "problems.id", "problems.scope", "problems.sourceItemId", "problems.origin.stageId", "createdAt"), &d.Baselines) + return read(s.queryCollection("reworkBaselines"), f, fields("_id", "itemId", "round", "stageId", "reviewOperationId", "problems.id", "problems.scope", "problems.sourceItemId", "problems.origin.stageId", "createdAt"), &d.Baselines) }) g.Go(func() error { - return readDirectory("items", s.db.Collection("reworkHandlings"), "itemId", itemIDs, fields("_id", "itemId", "round", "operationId", "skippedOpinionIds", "updatedAt"), &d.Handlings) + return readDirectory("items", s.queryCollection("reworkHandlings"), "itemId", itemIDs, fields("_id", "itemId", "round", "operationId", "skippedOpinionIds", "updatedAt"), &d.Handlings) }) err := g.Wait() d.SourceStats = &workforcestats.OutputSourceStats{Bytes: totalBytes, Documents: totalDocs} diff --git a/internal/store/annotation_output_snapshots.go b/internal/store/annotation_output_snapshots.go index b6db1d70..c3dbf2ba 100644 --- a/internal/store/annotation_output_snapshots.go +++ b/internal/store/annotation_output_snapshots.go @@ -34,13 +34,18 @@ func (s *MemoryStore) OutputSnapshotBlob(c context.Context, g outputsnapshot.Gen // These collections contain derived statistics only. No startup migration or // customer document is changed. Fixed slot _ids bound retained payload storage; // only the small coordinator is renewed. The normal Mongo _id indexes suffice. -func (s *MongoStore) outputCoordinator() *mongo.Collection { - return s.db.Collection(outputsnapshot.StateCollection) +func (s *MongoStore) outputCoordinator() mongoCollection { + return s.queryCollection(outputsnapshot.StateCollection) } -func (s *MongoStore) outputBlobs() *mongo.Collection { - return s.db.Collection(outputsnapshot.SlotsCollection) +func (s *MongoStore) outputBlobs() mongoCollection { + return s.queryCollection(outputsnapshot.SlotsCollection) } func (s *MongoStore) OutputSnapshotState(ctx context.Context) (outputsnapshot.State, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return outputsnapshot.State{}, err + } else if replica != nil { + return replica.OutputSnapshotState(ctx) + } var st outputsnapshot.State err := s.outputCoordinator().FindOne(ctx, bson.M{"_id": "global"}).Decode(&st) if err == mongo.ErrNoDocuments { @@ -52,6 +57,11 @@ func outputLeaseFilter(l outputsnapshot.Lease, now time.Time) bson.M { return bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch, "until": bson.M{"$gt": now}} } func (s *MongoStore) AcquireOutputSnapshot(ctx context.Context, key string, now time.Time) (outputsnapshot.Lease, bool, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return outputsnapshot.Lease{}, false, err + } else if replica != nil { + return replica.AcquireOutputSnapshot(ctx, key, now) + } owner := domain.NewID("output") prefix := "heads." + key filter := bson.M{"_id": "global", "$and": bson.A{ @@ -77,10 +87,20 @@ func (s *MongoStore) AcquireOutputSnapshot(ctx context.Context, key string, now return l, err == nil && res.MatchedCount == 1, err } func (s *MongoStore) RenewOutputSnapshot(ctx context.Context, l outputsnapshot.Lease, now time.Time) (bool, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return false, err + } else if replica != nil { + return replica.RenewOutputSnapshot(ctx, l, now) + } res, err := s.outputCoordinator().UpdateOne(ctx, outputLeaseFilter(l, now), bson.M{"$set": bson.M{"until": now.Add(outputsnapshot.LeaseDuration)}}) return err == nil && res.MatchedCount == 1, err } func (s *MongoStore) PublishOutputSnapshot(ctx context.Context, l outputsnapshot.Lease, g outputsnapshot.Generation, data []byte, now time.Time) (bool, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return false, err + } else if replica != nil { + return replica.PublishOutputSnapshot(ctx, l, g, data, now) + } if !outputsnapshot.ValidPublication(l, g, data) { return false, outputsnapshot.ErrCapacity } @@ -110,10 +130,20 @@ func (s *MongoStore) PublishOutputSnapshot(ctx context.Context, l outputsnapshot return err == nil && res.MatchedCount == 1, err } func (s *MongoStore) FailOutputSnapshot(ctx context.Context, l outputsnapshot.Lease, now time.Time, failure outputsnapshot.Failure) error { + if replica, err := s.reportReplica(ctx); err != nil { + return err + } else if replica != nil { + return replica.FailOutputSnapshot(ctx, l, now, failure) + } _, err := s.outputCoordinator().UpdateOne(ctx, bson.M{"_id": "global", "owner": l.Owner, "epoch": l.Epoch}, bson.M{"$inc": bson.M{"heads." + l.Key + ".consecutiveFailures": 1}, "$set": bson.M{"heads." + l.Key + ".lastFailure": failure, "heads." + l.Key + ".failed": true, "heads." + l.Key + ".nextAttempt": now.Add(outputsnapshot.RetryInterval), "owner": "", "until": now}}) return err } func (s *MongoStore) OutputSnapshotBlob(ctx context.Context, g outputsnapshot.Generation) ([]byte, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return nil, err + } else if replica != nil { + return replica.OutputSnapshotBlob(ctx, g) + } key := queryserving.Key("output-blob/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, g.SHA256) if cache := s.SharedQueryCache(); cache != nil { if b, hit := cache.Get(ctx, key); hit { @@ -139,6 +169,11 @@ func outputShardID(slot int, epoch int64, index int) string { return fmt.Sprintf("of-shard:%02d:%020d:%09d", slot, epoch, index) } func (s *MongoStore) StageOutputSnapshotShard(ctx context.Context, l outputsnapshot.Lease, index int, data []byte, now time.Time) (bool, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return false, err + } else if replica != nil { + return replica.StageOutputSnapshotShard(ctx, l, index, data, now) + } if l.Slot < 0 || l.Slot >= outputsnapshot.SlotCount || index < 0 || index >= outputsnapshot.MaxShards || len(data) > outputsnapshot.MaxEncodedBytes { return false, outputsnapshot.ErrCapacity } @@ -163,6 +198,11 @@ func (s *MongoStore) StageOutputSnapshotShard(ctx context.Context, l outputsnaps return err == nil, err } func (s *MongoStore) OutputSnapshotShard(ctx context.Context, g outputsnapshot.Generation, index int) ([]byte, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return nil, err + } else if replica != nil { + return replica.OutputSnapshotShard(ctx, g, index) + } key := queryserving.Key("output-shard/v1", s.db.Name(), g.ID, g.Slot, g.Epoch, index) if cache := s.SharedQueryCache(); cache != nil { if b, hit := cache.Get(ctx, key); hit { diff --git a/internal/store/annotation_output_stream.go b/internal/store/annotation_output_stream.go index 46bf9701..1bb952cf 100644 --- a/internal/store/annotation_output_stream.go +++ b/internal/store/annotation_output_stream.go @@ -582,6 +582,11 @@ func closeOutputWriters(writers map[string]*bufio.Writer, files map[string]*os.F // StreamAnnotationOutput preserves the old source interpreter and API contracts, // but replaces its global history-size cliff with per-part bounded partitions. func (s *MongoStore) StreamAnnotationOutput(ctx context.Context, org string, projects []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return nil, err + } else if replica != nil { + return replica.StreamAnnotationOutput(ctx, org, projects, yield) + } if len(projects) == 0 { return &workforcestats.OutputSourceStats{}, nil } diff --git a/internal/store/annotation_processing.go b/internal/store/annotation_processing.go index b07b8a2a..5cedcb23 100644 --- a/internal/store/annotation_processing.go +++ b/internal/store/annotation_processing.go @@ -71,10 +71,10 @@ func (s *MemoryStore) PutAnnotationTimelineIndex(_ context.Context, v domain.Ann return nil } func (s *MongoStore) GetAnnotationTimelineIndex(ctx context.Context, id string) (*domain.AnnotationTimelineIndex, error) { - return mongoGetByID[domain.AnnotationTimelineIndex](ctx, s.db.Collection("annotationTimelineIndexes"), id) + return mongoGetByID[domain.AnnotationTimelineIndex](ctx, s.queryCollection("annotationTimelineIndexes"), id) } func (s *MongoStore) PutAnnotationTimelineIndex(ctx context.Context, v domain.AnnotationTimelineIndex) error { - return mongoUpsert(ctx, s.db.Collection("annotationTimelineIndexes"), v.ID, v) + return mongoUpsert(ctx, s.queryCollection("annotationTimelineIndexes"), v.ID, v) } func annotationIndexJob(t domain.AnnotationTaskRecord) domain.AnnotationIndexJob { @@ -92,8 +92,8 @@ func (s *MemoryStore) EnqueueAnnotationIndex(_ context.Context, t domain.Annotat return nil } func (s *MongoStore) EnqueueAnnotationIndex(ctx context.Context, t domain.AnnotationTaskRecord) error { - return supplierGuardedMongoWrite(ctx, s.db.Collection("annotationIndexJobs"), t.ID, annotationIndexJob(t), func(ctx context.Context, d bson.M) error { - _, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": t.ID}, bson.M{"$setOnInsert": d}, options.Update().SetUpsert(true)) + return supplierGuardedMongoWrite(ctx, s.queryCollection("annotationIndexJobs"), t.ID, annotationIndexJob(t), func(ctx context.Context, d bson.M) error { + _, e := s.queryCollection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": t.ID}, bson.M{"$setOnInsert": d}, options.Update().SetUpsert(true)) return e }) } @@ -137,12 +137,12 @@ func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until // The bounded prefix range also works with the existing unique id index; // no schema migration or historical record rewrite is required. filter["id"] = bson.M{"$gte": "source-v1:", "$lt": "source-v1;"} - e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j) + e := s.queryCollection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j) if e != mongo.ErrNoDocuments { return &j, e } delete(filter, "id") - e = s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j) + e = s.queryCollection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j) if e == mongo.ErrNoDocuments { return nil, nil } @@ -160,7 +160,7 @@ func (s *MemoryStore) FinishAnnotationIndex(_ context.Context, j domain.Annotati } func (s *MongoStore) FinishAnnotationIndex(ctx context.Context, j domain.AnnotationIndexJob) (bool, error) { result, e := s.withSupplierProjectLocks(ctx, []string{j.ProjectID}, func(ctx context.Context) (any, error) { - r, e := s.db.Collection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j) + r, e := s.queryCollection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j) if e != nil { return false, e } diff --git a/internal/store/annotation_progress_reads.go b/internal/store/annotation_progress_reads.go index 1cd91182..d6be3d81 100644 --- a/internal/store/annotation_progress_reads.go +++ b/internal/store/annotation_progress_reads.go @@ -89,7 +89,7 @@ func latestProgressPipeline(org string, ids []string, annotation bool) mongo.Pip {{Key: "$replaceRoot", Value: bson.M{"newRoot": "$record"}}}, } } -func listLatestProgress[T any](ctx context.Context, collection *mongo.Collection, org string, ids []string, annotation bool) ([]T, error) { +func listLatestProgress[T any](ctx context.Context, collection mongoCollection, org string, ids []string, annotation bool) ([]T, error) { unique := annotationTaskIDSet(ids) ids = make([]string, 0, len(unique)) for id := range unique { diff --git a/internal/store/annotation_review_indexes.go b/internal/store/annotation_review_indexes.go index b9652432..a7fff1b6 100644 --- a/internal/store/annotation_review_indexes.go +++ b/internal/store/annotation_review_indexes.go @@ -15,7 +15,7 @@ func (s *MongoStore) EnsureAnnotationReviewIndexes(ctx context.Context) error { return ensureAnnotationReviewIndexes(ctx, s.annotationReviews) } -func ensureAnnotationReviewIndexes(ctx context.Context, collection *mongo.Collection) error { +func ensureAnnotationReviewIndexes(ctx context.Context, collection mongoCollection) error { // A legacy missing field must compare equal to round zero, not Mongo null. if _, err := collection.UpdateMany(ctx, bson.M{"reviewRound": nil}, bson.M{"$set": bson.M{"reviewRound": 0}}); err != nil { return err diff --git a/internal/store/annotation_source_admission.go b/internal/store/annotation_source_admission.go index 8205e8c7..de6e9aac 100644 --- a/internal/store/annotation_source_admission.go +++ b/internal/store/annotation_source_admission.go @@ -368,7 +368,7 @@ func (s *MongoStore) RetryAnnotationSourceValidation(ctx context.Context, task d next := annotationIndexJob(task) next.RevalidateSource = verifySource || AnnotationSourceValidationRequired(task) next.SupersededSourceFingerprint = supersededFingerprint - return nil, mongoUpsert(ctx, s.db.Collection("annotationIndexJobs"), task.ID, next) + return nil, mongoUpsert(ctx, s.queryCollection("annotationIndexJobs"), task.ID, next) }) return err } @@ -401,7 +401,7 @@ func (s *MongoStore) FailAnnotationSourceValidation(ctx context.Context, task do if createdAt == "" { job := annotationIndexJob(task) job.Status, job.Error = "failed", message - r, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": task.ID}, bson.M{"$setOnInsert": job}, options.Update().SetUpsert(true)) + r, e := s.queryCollection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": task.ID}, bson.M{"$setOnInsert": job}, options.Update().SetUpsert(true)) if mongo.IsDuplicateKeyError(e) { return false, nil } @@ -410,7 +410,7 @@ func (s *MongoStore) FailAnnotationSourceValidation(ctx context.Context, task do } return r.UpsertedCount == 1, nil } - r, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": task.ID, "organizationId": task.OrganizationID, "projectId": task.ProjectID, "runId": task.RunID, "createdAt": createdAt, "status": "completed"}, bson.M{"$set": bson.M{"status": "failed", "error": message}}) + r, e := s.queryCollection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": task.ID, "organizationId": task.OrganizationID, "projectId": task.ProjectID, "runId": task.RunID, "createdAt": createdAt, "status": "completed"}, bson.M{"$set": bson.M{"status": "failed", "error": message}}) if e != nil { return false, e } diff --git a/internal/store/annotation_source_jobs.go b/internal/store/annotation_source_jobs.go index 74b440e2..84d17ddb 100644 --- a/internal/store/annotation_source_jobs.go +++ b/internal/store/annotation_source_jobs.go @@ -23,7 +23,7 @@ func (s *MemoryStore) TouchAnnotationIndex(_ context.Context, id, owner, until s return true, nil } func (s *MongoStore) TouchAnnotationIndex(ctx context.Context, id, owner, until string) (bool, error) { - r, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": id, "owner": owner, "status": "running"}, bson.M{"$set": bson.M{"leaseUntil": until}}) + r, e := s.queryCollection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": id, "owner": owner, "status": "running"}, bson.M{"$set": bson.M{"leaseUntil": until}}) if e != nil { return false, e } @@ -40,7 +40,7 @@ func (s *MemoryStore) GetAnnotationIndexJob(_ context.Context, id string) (*doma return &j, nil } func (s *MongoStore) GetAnnotationIndexJob(ctx context.Context, id string) (*domain.AnnotationIndexJob, error) { - return mongoGetByID[domain.AnnotationIndexJob](ctx, s.db.Collection("annotationIndexJobs"), id) + return mongoGetByID[domain.AnnotationIndexJob](ctx, s.queryCollection("annotationIndexJobs"), id) } func sourceIndexJob(t domain.AnnotationTaskRecord, fingerprint string) domain.AnnotationIndexJob { @@ -64,8 +64,8 @@ func (s *MemoryStore) EnqueueAnnotationSourceIndex(_ context.Context, t domain.A } func (s *MongoStore) EnqueueAnnotationSourceIndex(ctx context.Context, t domain.AnnotationTaskRecord, fingerprint string, retry bool) error { j := sourceIndexJob(t, fingerprint) - return supplierGuardedMongoWrite(ctx, s.db.Collection("annotationIndexJobs"), j.ID, j, func(ctx context.Context, d bson.M) error { - collection := s.db.Collection("annotationIndexJobs") + return supplierGuardedMongoWrite(ctx, s.queryCollection("annotationIndexJobs"), j.ID, j, func(ctx context.Context, d bson.M) error { + collection := s.queryCollection("annotationIndexJobs") statuses := bson.A{"completed"} if retry { statuses = append(statuses, "failed") diff --git a/internal/store/annotation_work_buffer.go b/internal/store/annotation_work_buffer.go index ce823aa4..3175e8d7 100644 --- a/internal/store/annotation_work_buffer.go +++ b/internal/store/annotation_work_buffer.go @@ -65,7 +65,7 @@ func (s *MemoryStore) AnnotationWorkBuffer(_ context.Context, org, user, project func (s *MongoStore) annotationBuffer(ctx context.Context, org, user, project string, capacity int, reserve *HumanClaimReceipt, summary ...bool) (AnnotationWorkBuffer, error) { compact := len(summary) > 0 && summary[0] key := bufferKey(org, user, project) - collection := s.db.Collection("annotationWorkBuffers") + collection := s.queryCollection("annotationWorkBuffers") for { if err := ctx.Err(); err != nil { return AnnotationWorkBuffer{}, err diff --git a/internal/store/client_movi_stats.go b/internal/store/client_movi_stats.go index 04515897..fa1bc02c 100644 --- a/internal/store/client_movi_stats.go +++ b/internal/store/client_movi_stats.go @@ -26,7 +26,7 @@ func (store *MongoStore) InsertClientMoviStat(ctx context.Context, record domain if record.ExpireAt.IsZero() { record.ExpireAt = record.CreatedAt.Add(domain.ClientMoviStatRetention) } - _, err := store.db.Collection(clientMoviStatsCollection).InsertOne(ctx, record) + _, err := store.queryCollection(clientMoviStatsCollection).InsertOne(ctx, record) return err } @@ -36,7 +36,7 @@ func (store *MongoStore) ListClientMoviStats(ctx context.Context, organizationID if limit <= 0 || limit > 1000 { limit = 200 } - cursor, err := store.db.Collection(clientMoviStatsCollection).Find( + cursor, err := store.queryCollection(clientMoviStatsCollection).Find( ctx, bson.M{"organizationId": organizationID}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(int64(limit)), diff --git a/internal/store/deleted_annotator.go b/internal/store/deleted_annotator.go index 87bfdfb0..347e4d52 100644 --- a/internal/store/deleted_annotator.go +++ b/internal/store/deleted_annotator.go @@ -84,7 +84,7 @@ func (s *MongoStore) ResetDeletedAnnotatorTask(ctx context.Context, task domain. task.AnnotationRevisionFloor = max(task.AnnotationRevisionFloor, current.AnnotationRevisionFloor, current.CurrentRevision) } filter := bson.M{"organizationId": normalizedOrganizationID(task.OrganizationID), "taskId": task.ID} - for _, col := range []*mongo.Collection{s.annotationRevisions, s.sliceRevisions} { + for _, col := range []mongoCollection{s.annotationRevisions, s.sliceRevisions} { var last struct { Revision int `bson:"revision"` } @@ -104,7 +104,7 @@ func (s *MongoStore) ResetDeletedAnnotatorTask(ctx context.Context, task domain. } } filter["$or"] = bson.A{bson.M{"annotationEpoch": bson.M{"$lt": task.AnnotationEpoch}}, bson.M{"annotationEpoch": bson.M{"$exists": false}}} - for _, col := range []*mongo.Collection{s.annotationRevisions, s.sliceRevisions} { + for _, col := range []mongoCollection{s.annotationRevisions, s.sliceRevisions} { if _, err := col.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"supersededByAccountDeletion": true}}); err != nil { return nil, err } @@ -114,14 +114,14 @@ func (s *MongoStore) ResetDeletedAnnotatorTask(ctx context.Context, task domain. return err } -func upsertAnnotationEpoch(ctx context.Context, col *mongo.Collection, id string, value any, epoch int) error { +func upsertAnnotationEpoch(ctx context.Context, col mongoCollection, id string, value any, epoch int) error { _, err := upsertAnnotationEpochInserted(ctx, col, id, value, epoch) return err } // Use the write result, rather than an unlocked existence read: concurrent // projection updates must not be mistaken for task creation. -func upsertAnnotationEpochInserted(ctx context.Context, col *mongo.Collection, id string, value any, epoch int) (bool, error) { +func upsertAnnotationEpochInserted(ctx context.Context, col mongoCollection, id string, value any, epoch int) (bool, error) { inserted := false err := supplierGuardedMongoWrite(ctx, col, id, value, func(ctx context.Context, d bson.M) error { result, err := col.ReplaceOne(ctx, bson.M{"id": id, "$or": bson.A{bson.M{"annotationEpoch": bson.M{"$lte": epoch}}, bson.M{"annotationEpoch": bson.M{"$exists": false}}}}, d, options.Replace().SetUpsert(true)) @@ -136,7 +136,7 @@ func upsertAnnotationEpochInserted(ctx context.Context, col *mongo.Collection, i return inserted, err } -func (s *MongoStore) upsertActiveAnnotationRevision(ctx context.Context, col *mongo.Collection, id, taskID string, revision, epoch int, value any) error { +func (s *MongoStore) upsertActiveAnnotationRevision(ctx context.Context, col mongoCollection, id, taskID string, revision, epoch int, value any) error { return supplierGuardedMongoWrite(ctx, col, id, value, func(ctx context.Context, d bson.M) error { // This read and write hold the same task stripe as reset; a pre-lock // parent read alone would leave a check/write race. diff --git a/internal/store/human_claim.go b/internal/store/human_claim.go index d4a1833b..b16d03a3 100644 --- a/internal/store/human_claim.go +++ b/internal/store/human_claim.go @@ -73,7 +73,7 @@ func (s *MemoryStore) GetHumanClaimReceipt(_ context.Context, key string) (*Huma } func (s *MongoStore) GetHumanClaimReceipt(ctx context.Context, key string) (*HumanClaimReceipt, error) { var r HumanClaimReceipt - err := s.db.Collection("humanClaimRequests").FindOne(ctx, bson.M{"id": key}).Decode(&r) + err := s.queryCollection("humanClaimRequests").FindOne(ctx, bson.M{"id": key}).Decode(&r) if err == mongo.ErrNoDocuments { return nil, nil } @@ -359,7 +359,7 @@ func (s *MongoStore) commitHumanClaimTransaction(ctx context.Context, c HumanCla if e = s.AppendHumanEvent(sc, c.Event); e != nil { return nil, e } - if _, e = s.db.Collection("humanClaimRequests").InsertOne(sc, c.Receipt); e != nil { + if _, e = s.queryCollection("humanClaimRequests").InsertOne(sc, c.Receipt); e != nil { return nil, e } item := c.Item diff --git a/internal/store/human_claim_mongo_test.go b/internal/store/human_claim_mongo_test.go index 09445658..10f8bdb2 100644 --- a/internal/store/human_claim_mongo_test.go +++ b/internal/store/human_claim_mongo_test.go @@ -39,7 +39,7 @@ func TestHumanClaimMongoStandalone(t *testing.T) { } }() s := NewMongoStore(db) - for _, collection := range []*mongo.Collection{s.projects, s.users, s.suppliers, s.projectMembers, s.annotationTasks, s.humanBatches, s.humanItems, s.humanEvents, s.humanStageAttempts, db.Collection("humanClaimRequests")} { + for _, collection := range []mongoCollection{s.projects, s.users, s.suppliers, s.projectMembers, s.annotationTasks, s.humanBatches, s.humanItems, s.humanEvents, s.humanStageAttempts, db.Collection("humanClaimRequests")} { if _, err := collection.Indexes().CreateOne(ctx, uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}})); err != nil { t.Fatal(err) } diff --git a/internal/store/human_claim_standalone.go b/internal/store/human_claim_standalone.go index 22985408..afdfdad1 100644 --- a/internal/store/human_claim_standalone.go +++ b/internal/store/human_claim_standalone.go @@ -16,7 +16,7 @@ import ( // HumanItem's versioned update is the ownership authority. The remaining writes // are idempotent projections, completed before the API reports success. func (s *MongoStore) commitHumanClaimStandalone(ctx context.Context, proposed HumanClaimCommit) (*domain.HumanItemRecord, error) { - requests := s.db.Collection("humanClaimRequests") + requests := s.queryCollection("humanClaimRequests") r, err := s.GetHumanClaimReceipt(ctx, proposed.Receipt.ID) if err != nil { return nil, err diff --git a/internal/store/human_loop.go b/internal/store/human_loop.go index 5d0ef4a1..3543490f 100644 --- a/internal/store/human_loop.go +++ b/internal/store/human_loop.go @@ -861,6 +861,13 @@ func (store *MongoStore) UpsertHumanBatch(ctx context.Context, item domain.Human } func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) { + if query.PoolCandidateProjection { + if target, err := store.poolReplica(ctx); err != nil { + return nil, err + } else if target != nil { + return store.replicaPoolCandidates(ctx, target, organizationID, query) + } + } if query.NeedsReworkRouting { if err := store.validateDeletedAuthorRoutingIndexes(ctx); err != nil { return nil, err @@ -1082,6 +1089,22 @@ func (store *MongoStore) UpsertReleasePartition(ctx context.Context, item domain // ListHumanBatchAccessRecords omits large batch payloads. The projection is only // for list authorization/stage checks, never for updating a batch. func (store *MongoStore) ListHumanBatchAccessRecords(ctx context.Context, organizationID string, ids []string) ([]domain.HumanBatchRecord, error) { + if b, ok := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier); ok && b.counts { + if target, err := store.poolReplica(ctx); err != nil { + return nil, err + } else if target != nil { + return store.replicaBatchAccess(ctx, target, organizationID, ids) + } + } + return store.listAuthoritativeBatchAccess(ctx, organizationID, ids) +} +func (store *MongoStore) listAuthoritativeBatchAccess(ctx context.Context, organizationID string, ids []string) ([]domain.HumanBatchRecord, error) { + if store.queryReplicaFactStore { + return store.cachedReplicaBatchAccess(ctx, organizationID, ids) + } + return store.readBatchAccess(ctx, organizationID, ids) +} +func (store *MongoStore) readBatchAccess(ctx context.Context, organizationID string, ids []string) ([]domain.HumanBatchRecord, error) { if len(ids) == 0 { return []domain.HumanBatchRecord{}, nil } @@ -1134,6 +1157,11 @@ func (store *MemoryStore) ListReviewedHumanItemIDs(_ context.Context, organizati return ids, nil } func (store *MongoStore) ListReviewedHumanItemIDs(ctx context.Context, organizationID, actorID string) ([]string, error) { + if target, err := store.poolReplica(ctx); err != nil { + return nil, err + } else if target != nil { + return target.ListReviewedHumanItemIDs(ctx, organizationID, actorID) + } values, err := store.humanEvents.Distinct(ctx, "itemId", bson.M{"organizationId": normalizedOrganizationID(organizationID), "actorUserId": actorID, "eventType": bson.M{"$in": []string{"approved", "rejected", "combined_quality_approved", "combined_quality_rejected"}}}) if err != nil { return nil, err diff --git a/internal/store/human_pool_count_scan.go b/internal/store/human_pool_count_scan.go index a2a52e7c..20bfa0db 100644 --- a/internal/store/human_pool_count_scan.go +++ b/internal/store/human_pool_count_scan.go @@ -18,6 +18,17 @@ const humanPoolCountBatchSize = 1000 // existing filter; this visitor is not an authorized total or a public list. // Search/category counts use the same exhaustive matcher without detail hydration. func (s *MongoStore) VisitHumanPoolCountCandidates(ctx context.Context, org string, q domain.HumanItemQuery, visit func([]domain.HumanItemRecord) error) error { + if target, err := s.poolReplica(ctx); err != nil { + return err + } else if target != nil { + return s.visitReplicaPoolCounts(ctx, target, org, q, visit) + } + if s.queryReplicaFactStore { + return s.visitCachedReplicaCounts(ctx, org, q, visit) + } + return s.visitUncachedHumanPoolCounts(ctx, org, q, visit) +} +func (s *MongoStore) visitUncachedHumanPoolCounts(ctx context.Context, org string, q domain.HumanItemQuery, visit func([]domain.HumanItemRecord) error) error { if humanPoolFiltered(q) { return s.visitFilteredHumanPoolCountCandidates(ctx, org, q, visit) } @@ -32,7 +43,7 @@ func (s *MongoStore) VisitHumanPoolCountCandidates(ctx context.Context, org stri // A facet boundary keeps null/missing-value predicates above the covered // projection. Limit each facet to one bounded page, never the full pool. // Routing scans use additional fields and retain their existing path. - if q.BlindAnnotationActorID == "" && !q.NeedsReworkRouting { + if !s.queryReplicaFactStore && q.BlindAnnotationActorID == "" && !q.NeedsReworkRouting { used, err := s.visitCoveredHumanPoolCounts(ctx, org, q, filter, visit) if used || err != nil { return err diff --git a/internal/store/human_pool_search_discovery.go b/internal/store/human_pool_search_discovery.go index 2c59199a..e945e1ab 100644 --- a/internal/store/human_pool_search_discovery.go +++ b/internal/store/human_pool_search_discovery.go @@ -178,7 +178,7 @@ func (s *MongoStore) discoverIndexedHumanPoolSearch(ctx context.Context, org str pattern := poolSearchFoldedLiteral(token) + "|%[0-9a-f]{2}" ids := func(readCtx context.Context, name string, filter bson.M, index string) ([]string, bool, error) { filter = bson.M{"$and": bson.A{bson.M{"organizationId": normalizedOrganizationID(org)}, filter}} - c, e := poolSearchFind(readCtx, s.db.Collection(name), filter, bson.M{"_id": 0, "id": 1}, index, poolSearchReferenceLimit+1) + c, e := poolSearchFind(readCtx, s.queryCollection(name), filter, bson.M{"_id": 0, "id": 1}, index, poolSearchReferenceLimit+1) if e != nil { return nil, false, e } diff --git a/internal/store/human_pool_search_indexes.go b/internal/store/human_pool_search_indexes.go index 0a3ba9b9..481804c6 100644 --- a/internal/store/human_pool_search_indexes.go +++ b/internal/store/human_pool_search_indexes.go @@ -69,7 +69,7 @@ func poolSearchHintMissing(err error) bool { return errors.As(err, &command) && (command.Code == 27 || command.Code == 2 && strings.Contains(command.Message, "hint") && strings.Contains(command.Message, "index")) } -func poolSearchFind(ctx context.Context, c *mongo.Collection, filter, projection bson.M, index string, limit int64) (*mongo.Cursor, error) { +func poolSearchFind(ctx context.Context, c mongoCollection, filter, projection bson.M, index string, limit int64) (*mongo.Cursor, error) { opts := options.Find().SetProjection(projection).SetLimit(limit).SetMaxTime(8 * time.Second).SetBatchSize(poolSearchReferenceLimit + 1) if index != "" { opts.SetHint(index) @@ -81,7 +81,7 @@ func poolSearchFind(ctx context.Context, c *mongo.Collection, filter, projection } return cur, err } -func poolSearchAggregate(ctx context.Context, c *mongo.Collection, p mongo.Pipeline) (*mongo.Cursor, error) { +func poolSearchAggregate(ctx context.Context, c mongoCollection, p mongo.Pipeline) (*mongo.Cursor, error) { opts := options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8 * time.Second).SetHint(poolGuardIndex) cur, err := c.Aggregate(ctx, p, opts) diff --git a/internal/store/human_pool_search_legacy.go b/internal/store/human_pool_search_legacy.go index 1019ca52..db928fa1 100644 --- a/internal/store/human_pool_search_legacy.go +++ b/internal/store/human_pool_search_legacy.go @@ -57,7 +57,7 @@ func (s *MongoStore) legacyHumanPoolSearchCandidates(ctx context.Context, org st if extra != nil { f["$and"] = append(f["$and"].(bson.A), extra) } - cur, err := s.db.Collection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetLimit(maxReferences+1).SetMaxTime(8*time.Second)) + cur, err := s.queryCollection(collection).Find(ctx, f, options.Find().SetProjection(bson.M{"_id": 0, "id": 1}).SetLimit(maxReferences+1).SetMaxTime(8*time.Second)) if err != nil { return nil, false, err } @@ -117,7 +117,7 @@ func (s *MongoStore) humanPoolSearchReferenceScope(ctx context.Context, org stri sets[name] = map[string]bool{} } read := func(collection string, filter, projection bson.M, visit func(bson.M)) (bool, error) { - cur, err := s.db.Collection(collection).Find(ctx, filter, options.Find().SetProjection(projection).SetLimit(int64(limit+1)).SetMaxTime(8*time.Second)) + cur, err := s.queryCollection(collection).Find(ctx, filter, options.Find().SetProjection(projection).SetLimit(int64(limit+1)).SetMaxTime(8*time.Second)) if err != nil { return false, err } diff --git a/internal/store/human_pool_search_scalar_names.go b/internal/store/human_pool_search_scalar_names.go index 9e759dca..c21e35b5 100644 --- a/internal/store/human_pool_search_scalar_names.go +++ b/internal/store/human_pool_search_scalar_names.go @@ -52,7 +52,7 @@ func (s *MongoStore) poolScalarNames(ctx context.Context, org, collection, index bson.D{{Key: "$unwind", Value: "$rows"}}, bson.D{{Key: "$replaceWith", Value: "$rows"}}, } - cur, err := s.db.Collection(collection).Aggregate(ctx, pipeline, options.Aggregate().SetHint(index).SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(poolSearchReferenceLimit+1)) + cur, err := s.queryCollection(collection).Aggregate(ctx, pipeline, options.Aggregate().SetHint(index).SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(poolSearchReferenceLimit+1)) if err != nil { return nil, err } diff --git a/internal/store/human_trace.go b/internal/store/human_trace.go index 7ae41950..6cc53060 100644 --- a/internal/store/human_trace.go +++ b/internal/store/human_trace.go @@ -91,7 +91,7 @@ func (s *MongoStore) ListHumanTraceSummaries(ctx context.Context, org string, it if len(ids) == 0 { return result, nil } - cur, err := s.db.Collection(collection).Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "name": 1})) + cur, err := s.queryCollection(collection).Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "name": 1})) if err != nil { return nil, err } diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 10fa0575..f2a6b498 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -9,6 +9,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "droboticflow/internal/domain" @@ -20,6 +21,11 @@ import ( ) type MongoStore struct { + querySyncEnabled atomic.Bool + queryReplicaReads *queryReplicaReads + queryReplicaFactStore bool + queryReplicaCounts queryCountCache + queryReplicaBatchFacts queryBatchFacts queryServing *queryServingOptions productionReadFlights singleflight.Group claimReadFlights singleflight.Group @@ -27,136 +33,134 @@ type MongoStore struct { claimReadOnce sync.Once claimReadSlots chan struct{} db *mongo.Database - organizations *mongo.Collection - projects *mongo.Collection - operators *mongo.Collection - builtinOperatorDeletions *mongo.Collection - workflows *mongo.Collection - workflowRuntimePolicies *mongo.Collection - templates *mongo.Collection - workflowReleases *mongo.Collection - projectWorkflowInstances *mongo.Collection - runs *mongo.Collection - sourceStates *mongo.Collection - captureRecords *mongo.Collection - collectionTasks *mongo.Collection - datasetVersions *mongo.Collection - dataMetricFacts *mongo.Collection - auditTasks *mongo.Collection - samplingTasks *mongo.Collection - annotationWorkgroups *mongo.Collection - annotationMemberships *mongo.Collection - suppliers *mongo.Collection - projectMembers *mongo.Collection - supplierProjectAuthorizations *mongo.Collection - supplierProjectAuthorizationInit *mongo.Collection - annotationLabels *mongo.Collection - projectAnnotationLabels *mongo.Collection - annotationTemplates *mongo.Collection - annotationTasks *mongo.Collection - annotationRevisions *mongo.Collection - sliceRevisions *mongo.Collection - annotationReviews *mongo.Collection - annotationWorklogEvents *mongo.Collection - organizationPlugins *mongo.Collection - humanBatches *mongo.Collection - humanItems *mongo.Collection - humanStageAttempts *mongo.Collection - humanEvents *mongo.Collection - humanRejects *mongo.Collection - releasePartitions *mongo.Collection - deliveryPackages *mongo.Collection - users *mongo.Collection - sessions *mongo.Collection - storageConnections *mongo.Collection - uploadSessions *mongo.Collection - mongoConnections *mongo.Collection - objectSyncSpaces *mongo.Collection - objectSyncDevices *mongo.Collection - objectSyncAuthorizations *mongo.Collection - objectSyncCredentials *mongo.Collection - objectSyncSessions *mongo.Collection - objectSyncItems *mongo.Collection - objectSyncLeases *mongo.Collection - objectSyncHeads *mongo.Collection - objectSyncVersions *mongo.Collection - objectSyncAuditEvents *mongo.Collection - objectSyncAuditSeals *mongo.Collection - objectSyncReleases *mongo.Collection - objectSyncClientReleases *mongo.Collection - objectSyncReconciles *mongo.Collection - objectSyncIdempotency *mongo.Collection - objectSyncRateLimits *mongo.Collection - settings *mongo.Collection + organizations mongoCollection + projects mongoCollection + operators mongoCollection + builtinOperatorDeletions mongoCollection + workflows mongoCollection + workflowRuntimePolicies mongoCollection + templates mongoCollection + workflowReleases mongoCollection + projectWorkflowInstances mongoCollection + runs mongoCollection + sourceStates mongoCollection + captureRecords mongoCollection + collectionTasks mongoCollection + datasetVersions mongoCollection + dataMetricFacts mongoCollection + auditTasks mongoCollection + samplingTasks mongoCollection + annotationWorkgroups mongoCollection + annotationMemberships mongoCollection + suppliers mongoCollection + projectMembers mongoCollection + supplierProjectAuthorizations mongoCollection + supplierProjectAuthorizationInit mongoCollection + annotationLabels mongoCollection + projectAnnotationLabels mongoCollection + annotationTemplates mongoCollection + annotationTasks mongoCollection + annotationRevisions mongoCollection + sliceRevisions mongoCollection + annotationReviews mongoCollection + annotationWorklogEvents mongoCollection + organizationPlugins mongoCollection + humanBatches mongoCollection + humanItems mongoCollection + humanStageAttempts mongoCollection + humanEvents mongoCollection + humanRejects mongoCollection + releasePartitions mongoCollection + deliveryPackages mongoCollection + users mongoCollection + sessions mongoCollection + storageConnections mongoCollection + uploadSessions mongoCollection + mongoConnections mongoCollection + objectSyncSpaces mongoCollection + objectSyncDevices mongoCollection + objectSyncAuthorizations mongoCollection + objectSyncCredentials mongoCollection + objectSyncSessions mongoCollection + objectSyncItems mongoCollection + objectSyncLeases mongoCollection + objectSyncHeads mongoCollection + objectSyncVersions mongoCollection + objectSyncAuditEvents mongoCollection + objectSyncAuditSeals mongoCollection + objectSyncReleases mongoCollection + objectSyncClientReleases mongoCollection + objectSyncReconciles mongoCollection + objectSyncIdempotency mongoCollection + objectSyncRateLimits mongoCollection + settings mongoCollection runSummaryAggregates *runSummaryAggregateCache } func NewMongoStore(db *mongo.Database) *MongoStore { - return &MongoStore{ - db: db, - organizations: db.Collection("organizations"), - projects: db.Collection("projects"), - operators: db.Collection("operators"), - builtinOperatorDeletions: db.Collection("operatorTombstones"), - workflows: db.Collection("workflows"), - workflowRuntimePolicies: db.Collection("workflowRuntimePolicies"), - templates: db.Collection("templates"), - workflowReleases: db.Collection("workflowReleases"), - projectWorkflowInstances: db.Collection("projectWorkflowInstances"), - runs: db.Collection("runs"), - sourceStates: db.Collection("sourceStates"), - captureRecords: db.Collection("captureRecords"), - collectionTasks: db.Collection("collectionTasks"), - datasetVersions: db.Collection("datasetVersions"), - dataMetricFacts: db.Collection("dataMetricFacts"), - auditTasks: db.Collection("auditTasks"), - samplingTasks: db.Collection("samplingTasks"), - annotationWorkgroups: db.Collection("annotationWorkgroups"), - annotationMemberships: db.Collection("annotationMemberships"), - suppliers: db.Collection("suppliers"), - projectMembers: db.Collection("projectMembers"), - supplierProjectAuthorizations: db.Collection("supplierProjectAuthorizations"), - supplierProjectAuthorizationInit: db.Collection("supplierProjectAuthorizationInit"), - annotationLabels: db.Collection("annotationLabels"), - projectAnnotationLabels: db.Collection("projectAnnotationLabels"), - annotationTemplates: db.Collection("annotationTemplates"), - annotationTasks: db.Collection("annotationTasks"), - annotationRevisions: db.Collection("annotationRevisions"), - sliceRevisions: db.Collection("sliceRevisions"), - annotationReviews: db.Collection("annotationReviews"), - annotationWorklogEvents: db.Collection("annotationWorklogEvents"), - organizationPlugins: db.Collection("organizationPlugins"), - humanBatches: db.Collection("humanBatches"), - humanItems: db.Collection("humanItems"), - humanStageAttempts: db.Collection("humanStageAttempts"), - humanEvents: db.Collection("humanEvents"), - humanRejects: db.Collection("humanRejects"), - releasePartitions: db.Collection("humanReleasePartitions"), - deliveryPackages: db.Collection("deliveryPackages"), - users: db.Collection("users"), - sessions: db.Collection("sessions"), - storageConnections: db.Collection("storageConnections"), - uploadSessions: db.Collection("objectStorageUploadSessions"), - mongoConnections: db.Collection("mongoConnections"), - objectSyncSpaces: db.Collection("objectSyncSpaces"), - objectSyncDevices: db.Collection("objectSyncDevices"), - objectSyncAuthorizations: db.Collection("objectSyncAuthorizations"), - objectSyncCredentials: db.Collection("objectSyncCredentials"), - objectSyncSessions: db.Collection("objectSyncSessions"), - objectSyncItems: db.Collection("objectSyncItems"), - objectSyncLeases: db.Collection("objectSyncLeases"), - objectSyncHeads: db.Collection("objectSyncHeads"), - objectSyncVersions: db.Collection("objectSyncVersions"), - objectSyncAuditEvents: db.Collection("objectSyncAuditEvents"), - objectSyncAuditSeals: db.Collection("objectSyncAuditSeals"), - objectSyncReleases: db.Collection("objectSyncReleases"), - objectSyncClientReleases: db.Collection("objectSyncClientReleases"), - objectSyncReconciles: db.Collection("objectSyncReconciles"), - objectSyncIdempotency: db.Collection("objectSyncIdempotency"), - objectSyncRateLimits: db.Collection("objectSyncRateLimits"), - settings: db.Collection("settings"), - runSummaryAggregates: newRunSummaryAggregateCache(runSummaryAggregateCacheTTL, runSummaryAggregateCacheMaxEntries), - } + out := &MongoStore{db: db, runSummaryAggregates: newRunSummaryAggregateCache(runSummaryAggregateCacheTTL, runSummaryAggregateCacheMaxEntries)} + out.organizations = out.queryCollection("organizations") + out.projects = out.queryCollection("projects") + out.operators = out.queryCollection("operators") + out.builtinOperatorDeletions = out.queryCollection("operatorTombstones") + out.workflows = out.queryCollection("workflows") + out.workflowRuntimePolicies = out.queryCollection("workflowRuntimePolicies") + out.templates = out.queryCollection("templates") + out.workflowReleases = out.queryCollection("workflowReleases") + out.projectWorkflowInstances = out.queryCollection("projectWorkflowInstances") + out.runs = out.queryCollection("runs") + out.sourceStates = out.queryCollection("sourceStates") + out.captureRecords = out.queryCollection("captureRecords") + out.collectionTasks = out.queryCollection("collectionTasks") + out.datasetVersions = out.queryCollection("datasetVersions") + out.dataMetricFacts = out.queryCollection("dataMetricFacts") + out.auditTasks = out.queryCollection("auditTasks") + out.samplingTasks = out.queryCollection("samplingTasks") + out.annotationWorkgroups = out.queryCollection("annotationWorkgroups") + out.annotationMemberships = out.queryCollection("annotationMemberships") + out.suppliers = out.queryCollection("suppliers") + out.projectMembers = out.queryCollection("projectMembers") + out.supplierProjectAuthorizations = out.queryCollection("supplierProjectAuthorizations") + out.supplierProjectAuthorizationInit = out.queryCollection("supplierProjectAuthorizationInit") + out.annotationLabels = out.queryCollection("annotationLabels") + out.projectAnnotationLabels = out.queryCollection("projectAnnotationLabels") + out.annotationTemplates = out.queryCollection("annotationTemplates") + out.annotationTasks = out.queryCollection("annotationTasks") + out.annotationRevisions = out.queryCollection("annotationRevisions") + out.sliceRevisions = out.queryCollection("sliceRevisions") + out.annotationReviews = out.queryCollection("annotationReviews") + out.annotationWorklogEvents = out.queryCollection("annotationWorklogEvents") + out.organizationPlugins = out.queryCollection("organizationPlugins") + out.humanBatches = out.queryCollection("humanBatches") + out.humanItems = out.queryCollection("humanItems") + out.humanStageAttempts = out.queryCollection("humanStageAttempts") + out.humanEvents = out.queryCollection("humanEvents") + out.humanRejects = out.queryCollection("humanRejects") + out.releasePartitions = out.queryCollection("humanReleasePartitions") + out.deliveryPackages = out.queryCollection("deliveryPackages") + out.users = out.queryCollection("users") + out.sessions = out.queryCollection("sessions") + out.storageConnections = out.queryCollection("storageConnections") + out.uploadSessions = out.queryCollection("objectStorageUploadSessions") + out.mongoConnections = out.queryCollection("mongoConnections") + out.objectSyncSpaces = out.queryCollection("objectSyncSpaces") + out.objectSyncDevices = out.queryCollection("objectSyncDevices") + out.objectSyncAuthorizations = out.queryCollection("objectSyncAuthorizations") + out.objectSyncCredentials = out.queryCollection("objectSyncCredentials") + out.objectSyncSessions = out.queryCollection("objectSyncSessions") + out.objectSyncItems = out.queryCollection("objectSyncItems") + out.objectSyncLeases = out.queryCollection("objectSyncLeases") + out.objectSyncHeads = out.queryCollection("objectSyncHeads") + out.objectSyncVersions = out.queryCollection("objectSyncVersions") + out.objectSyncAuditEvents = out.queryCollection("objectSyncAuditEvents") + out.objectSyncAuditSeals = out.queryCollection("objectSyncAuditSeals") + out.objectSyncReleases = out.queryCollection("objectSyncReleases") + out.objectSyncClientReleases = out.queryCollection("objectSyncClientReleases") + out.objectSyncReconciles = out.queryCollection("objectSyncReconciles") + out.objectSyncIdempotency = out.queryCollection("objectSyncIdempotency") + out.objectSyncRateLimits = out.queryCollection("objectSyncRateLimits") + out.settings = out.queryCollection("settings") + return out } func (store *MongoStore) Ready(ctx context.Context) error { @@ -283,7 +287,7 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin return false, nil } filter := orgCompatibilityFilter(organizationID) - for _, collection := range []*mongo.Collection{ + for _, collection := range []mongoCollection{ store.users, store.sessions, store.projects, @@ -439,7 +443,7 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er if len(taskIDs) > 0 { taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}} - for _, collection := range []*mongo.Collection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} { + for _, collection := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} { if _, err := collection.DeleteMany(ctx, taskFilter); err != nil { return false, err } @@ -447,7 +451,7 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er } if len(itemIDs) > 0 { itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}} - for _, collection := range []*mongo.Collection{store.humanStageAttempts, store.humanEvents} { + for _, collection := range []mongoCollection{store.humanStageAttempts, store.humanEvents} { if _, err := collection.DeleteMany(ctx, itemFilter); err != nil { return false, err } @@ -460,12 +464,12 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er } for _, operation := range []struct { - collection *mongo.Collection + collection mongoCollection filter bson.M }{ {store.annotationTasks, bson.M{"projectId": id}}, - {store.db.Collection("annotationTimelineIndexes"), bson.M{"projectId": id}}, - {store.db.Collection("annotationIndexJobs"), bson.M{"projectId": id}}, + {store.queryCollection("annotationTimelineIndexes"), bson.M{"projectId": id}}, + {store.queryCollection("annotationIndexJobs"), bson.M{"projectId": id}}, {store.humanRejects, bson.M{"projectId": id}}, {store.humanItems, bson.M{"projectId": id}}, {store.humanBatches, bson.M{"projectId": id}}, @@ -772,7 +776,7 @@ func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, e } if len(taskIDs) > 0 { taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}} - for _, collection := range []*mongo.Collection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} { + for _, collection := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} { if _, err := collection.DeleteMany(ctx, taskFilter); err != nil { return false, err } @@ -780,7 +784,7 @@ func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, e } if len(itemIDs) > 0 { itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}} - for _, collection := range []*mongo.Collection{store.humanStageAttempts, store.humanEvents} { + for _, collection := range []mongoCollection{store.humanStageAttempts, store.humanEvents} { if _, err := collection.DeleteMany(ctx, itemFilter); err != nil { return false, err } @@ -792,13 +796,13 @@ func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, e } } for _, operation := range []struct { - collection *mongo.Collection + collection mongoCollection filter bson.M }{ {store.projectWorkflowInstances, bson.M{"workflowId": id}}, {store.annotationTasks, bson.M{"workflowId": id}}, - {store.db.Collection("annotationTimelineIndexes"), bson.M{"workflowId": id}}, - {store.db.Collection("annotationIndexJobs"), bson.M{"workflowId": id}}, + {store.queryCollection("annotationTimelineIndexes"), bson.M{"workflowId": id}}, + {store.queryCollection("annotationIndexJobs"), bson.M{"workflowId": id}}, {store.humanRejects, bson.M{"workflowId": id}}, {store.humanItems, bson.M{"workflowId": id}}, {store.humanBatches, bson.M{"workflowId": id}}, @@ -953,7 +957,7 @@ func (store *MongoStore) DeleteTemplate(ctx context.Context, id string) (bool, e // Removing the template entry must not invalidate immutable snapshots // still used by project instances or historical/active Runs. referenced := []any{} - for _, collection := range []*mongo.Collection{store.projectWorkflowInstances, store.runs} { + for _, collection := range []mongoCollection{store.projectWorkflowInstances, store.runs} { ids, err := collection.Distinct(ctx, "releaseId", bson.M{"releaseId": bson.M{"$exists": true, "$ne": ""}}) if err != nil { return false, err @@ -2715,7 +2719,7 @@ func mongoBrowserDocument(document bson.M) map[string]any { return sanitizeDatabaseBrowserDocument(result) } -func mongoBrowserSampleDocument(ctx context.Context, collection *mongo.Collection) (map[string]any, error) { +func mongoBrowserSampleDocument(ctx context.Context, collection mongoCollection) (map[string]any, error) { var sample bson.M err := collection.FindOne(ctx, bson.D{}, options.FindOne().SetSort(bson.D{{Key: "_id", Value: -1}})).Decode(&sample) if err == mongo.ErrNoDocuments { @@ -2727,7 +2731,7 @@ func mongoBrowserSampleDocument(ctx context.Context, collection *mongo.Collectio return mongoBrowserDocument(sample), nil } -func mongoBrowserCollectionSummary(ctx context.Context, collection *mongo.Collection, name string) (domain.DatabaseCollectionSummary, error) { +func mongoBrowserCollectionSummary(ctx context.Context, collection mongoCollection, name string) (domain.DatabaseCollectionSummary, error) { count, err := collection.EstimatedDocumentCount(ctx) if err != nil { count, err = collection.CountDocuments(ctx, bson.D{}) @@ -2769,7 +2773,7 @@ func mongoBrowserWritablePatch(document map[string]any) bson.M { return mongoBrowserFilter(writable).(bson.M) } -func mongoBrowserFindDocumentByFilter(ctx context.Context, collection *mongo.Collection, filter bson.M) (map[string]any, error) { +func mongoBrowserFindDocumentByFilter(ctx context.Context, collection mongoCollection, filter bson.M) (map[string]any, error) { document := bson.M{} if err := collection.FindOne(ctx, filter).Decode(&document); err != nil { if err == mongo.ErrNoDocuments { @@ -2811,7 +2815,7 @@ func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.Databa if IsPrivateDerivedCollection(name) { continue } - summary, err := mongoBrowserCollectionSummary(ctx, store.db.Collection(name), name) + summary, err := mongoBrowserCollectionSummary(ctx, store.queryCollection(name), name) if err != nil { return domain.DatabaseOverview{}, err } @@ -2837,7 +2841,7 @@ func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input doma skip = 0 } - collection := store.db.Collection(collectionName) + collection := store.queryCollection(collectionName) filter, _ := mongoBrowserFilter(input.Filter).(bson.M) if filter == nil { filter = bson.M{} @@ -2895,7 +2899,7 @@ func (store *MongoStore) CreateDatabaseCollection(ctx context.Context, name stri if err := store.db.CreateCollection(ctx, collectionName); err != nil { return domain.DatabaseCollectionSummary{}, err } - return mongoBrowserCollectionSummary(ctx, store.db.Collection(collectionName), collectionName) + return mongoBrowserCollectionSummary(ctx, store.queryCollection(collectionName), collectionName) } func (store *MongoStore) ListDatabaseIndexes(ctx context.Context, collectionName string) ([]domain.DatabaseIndexSummary, error) { @@ -2903,7 +2907,7 @@ func (store *MongoStore) ListDatabaseIndexes(ctx context.Context, collectionName if name == "" { return nil, fmt.Errorf("collection is required") } - cursor, err := store.db.Collection(name).Indexes().List(ctx) + cursor, err := store.queryCollection(name).Indexes().List(ctx) if err != nil { return nil, err } @@ -2947,7 +2951,7 @@ func (store *MongoStore) CreateDatabaseIndex(ctx context.Context, collectionName } model.Options = opts } - createdName, err := store.db.Collection(name).Indexes().CreateOne(ctx, model) + createdName, err := store.queryCollection(name).Indexes().CreateOne(ctx, model) if err != nil { return domain.DatabaseIndexSummary{}, err } @@ -2975,13 +2979,16 @@ func (store *MongoStore) DeleteDatabaseIndex(ctx context.Context, collectionName if target == "_id_" { return fmt.Errorf("index %s cannot be deleted", target) } - _, err := store.db.Collection(name).Indexes().DropOne(ctx, target) + _, err := store.queryCollection(name).Indexes().DropOne(ctx, target) return err } func (store *MongoStore) RenameDatabaseCollection(ctx context.Context, name string, newName string) (domain.DatabaseCollectionSummary, error) { collectionName := strings.TrimSpace(name) nextName := strings.TrimSpace(newName) + if store.querySyncEnabled.Load() && (queryReplicaFields[collectionName] != "" || queryReplicaFields[nextName] != "") { + return domain.DatabaseCollectionSummary{}, ErrQueryReplicaUnsupported + } if collectionName == "" { return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required") } @@ -2995,7 +3002,7 @@ func (store *MongoStore) RenameDatabaseCollection(ctx context.Context, name stri if err := store.db.Client().Database("admin").RunCommand(ctx, command).Err(); err != nil { return domain.DatabaseCollectionSummary{}, err } - return mongoBrowserCollectionSummary(ctx, store.db.Collection(nextName), nextName) + return mongoBrowserCollectionSummary(ctx, store.queryCollection(nextName), nextName) } func (store *MongoStore) DeleteDatabaseCollection(ctx context.Context, name string) error { @@ -3003,7 +3010,7 @@ func (store *MongoStore) DeleteDatabaseCollection(ctx context.Context, name stri if collectionName == "" { return fmt.Errorf("collection is required") } - if err := store.db.Collection(collectionName).Drop(ctx); err != nil { + if err := store.queryCollection(collectionName).Drop(ctx); err != nil { return err } return nil @@ -3014,7 +3021,7 @@ func (store *MongoStore) InsertDatabaseDocument(ctx context.Context, collectionN if name == "" { return nil, fmt.Errorf("collection is required") } - collection := store.db.Collection(name) + collection := store.queryCollection(name) payload := mongoBrowserWritableDocument(document) result, err := collection.InsertOne(ctx, payload) if err != nil { @@ -3032,7 +3039,7 @@ func (store *MongoStore) ReplaceDatabaseDocument(ctx context.Context, collection if err != nil { return nil, err } - collection := store.db.Collection(name) + collection := store.queryCollection(name) result, err := collection.ReplaceOne(ctx, filter, mongoBrowserWritableDocument(document)) if err != nil { return nil, err @@ -3052,7 +3059,7 @@ func (store *MongoStore) PatchDatabaseDocument(ctx context.Context, collectionNa if err != nil { return nil, err } - collection := store.db.Collection(name) + collection := store.queryCollection(name) result, err := collection.UpdateOne(ctx, filter, bson.M{"$set": mongoBrowserWritablePatch(patch)}) if err != nil { return nil, err @@ -3076,7 +3083,7 @@ func (store *MongoStore) UpsertDatabaseDocument(ctx context.Context, collectionN if len(setOnInsert) > 0 { update["$setOnInsert"] = mongoBrowserWritablePatch(setOnInsert) } - collection := store.db.Collection(name) + collection := store.queryCollection(name) result, err := collection.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true)) if err != nil { return nil, false, err @@ -3102,7 +3109,7 @@ func (store *MongoStore) DeleteDatabaseDocument(ctx context.Context, collectionN if err != nil { return err } - result, err := store.db.Collection(name).DeleteOne(ctx, filter) + result, err := store.queryCollection(name).DeleteOne(ctx, filter) if err != nil { return err } diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index f5af7164..0025e93d 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -44,7 +44,7 @@ func ttlIndex(name string, keys bson.D, expireAfterSeconds int32) mongo.IndexMod } } -func ensureCollectionIndexes(ctx context.Context, collection *mongo.Collection, models []mongo.IndexModel) error { +func ensureCollectionIndexes(ctx context.Context, collection mongoCollection, models []mongo.IndexModel) error { if len(models) == 0 { return nil } @@ -57,14 +57,14 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { for _, col := range []string{"users", "annotationTasks", "humanItems", "humanBatches", "runs", "workflows"} { models = append(models, index("supplier_deletion_"+col, bson.D{{Key: "plan.records." + col, Value: 1}})) } - if e := ensureCollectionIndexes(ctx, store.db.Collection("supplierDeletionTasks"), models); e != nil { + if e := ensureCollectionIndexes(ctx, store.queryCollection("supplierDeletionTasks"), models); e != nil { return e } if err := store.EnsureAnnotationReviewIndexes(ctx); err != nil { return err } - indexesByCollection := map[*mongo.Collection][]mongo.IndexModel{ + indexesByCollection := map[mongoCollection][]mongo.IndexModel{ store.supplierProjectAuthorizations: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organization_project_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "status", Value: 1}}), @@ -73,17 +73,17 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { store.supplierProjectAuthorizationInit: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), }, - store.db.Collection("annotationTimelineIndexes"): { + store.queryCollection("annotationTimelineIndexes"): { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("projectId", bson.D{{Key: "projectId", Value: 1}}), index("workflowId", bson.D{{Key: "workflowId", Value: 1}}), }, - store.db.Collection("annotationIndexJobs"): { + store.queryCollection("annotationIndexJobs"): { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("status_nextAttemptAt", bson.D{{Key: "status", Value: 1}, {Key: "nextAttemptAt", Value: 1}}), index("status_leaseUntil", bson.D{{Key: "status", Value: 1}, {Key: "leaseUntil", Value: 1}}), }, - store.db.Collection("humanClaimRequests"): { + store.queryCollection("humanClaimRequests"): { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organization_project", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}}), }, @@ -426,7 +426,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { store.projectAnnotationLabels: { uniqueIndex("projectId_unique", bson.D{{Key: "projectId", Value: 1}}), }, - store.db.Collection(clientMoviStatsCollection): { + store.queryCollection(clientMoviStatsCollection): { ttlIndex("expireAt_ttl", bson.D{{Key: "expireAt", Value: 1}}, int32(domain.ClientMoviStatRetention/time.Second)), index("organization_created", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}}), index("kind", bson.D{{Key: "kind", Value: 1}}), @@ -449,7 +449,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { // mislabels as "phone must be unique" since the caller only checks whether // *some* unique index on the collection conflicted. Missing-index errors // are ignored since that's the desired end state. -func dropObsoleteIndexes(ctx context.Context, collection *mongo.Collection, names ...string) error { +func dropObsoleteIndexes(ctx context.Context, collection mongoCollection, names ...string) error { for _, name := range names { if _, err := collection.Indexes().DropOne(ctx, name); err != nil { var cmdErr mongo.CommandError diff --git a/internal/store/mongo_migrations.go b/internal/store/mongo_migrations.go index 518d5fcb..3d497c74 100644 --- a/internal/store/mongo_migrations.go +++ b/internal/store/mongo_migrations.go @@ -257,7 +257,7 @@ func (store *MongoStore) migrateLegacyWorkgroupsToProjectMembers(ctx context.Con if err := store.migrateAnnotationSupplierHyperparams(ctx, groupSupplier); err != nil { return err } - collections := []*mongo.Collection{store.annotationTasks, store.annotationWorklogEvents, store.humanItems, store.deliveryPackages} + collections := []mongoCollection{store.annotationTasks, store.annotationWorklogEvents, store.humanItems, store.deliveryPackages} for _, collection := range collections { pipeline := mongo.Pipeline{bson.D{{Key: "$set", Value: bson.D{ {Key: "supplierId", Value: bson.D{{Key: "$ifNull", Value: bson.A{"$vendorId", ""}}}}, @@ -335,7 +335,7 @@ func (store *MongoStore) migrateLegacyWorkgroupsToProjectMembers(ctx context.Con } func (store *MongoStore) dropLegacyWorkgroupIndexes(ctx context.Context) error { - legacyIndexes := map[*mongo.Collection][]string{ + legacyIndexes := map[mongoCollection][]string{ store.annotationTasks: {"organizationId_projectId_workgroupId_status"}, store.annotationWorklogEvents: {"organizationId_vendorId_workgroupId_userId_createdAt"}, store.humanItems: {"organizationId_vendorId_workgroupId_stage_status_assignee"}, @@ -421,7 +421,7 @@ func (store *MongoStore) migrateAnnotationSupplierHyperparams(ctx context.Contex } func (store *MongoStore) archiveAndDropLegacyWorkgroupCollections(ctx context.Context, groups []domain.AnnotationWorkgroupRecord, memberships []domain.ProjectAnnotationMembershipRecord) error { - audit := store.db.Collection("legacyWorkgroupAudit") + audit := store.queryCollection("legacyWorkgroupAudit") archive := func(source string, id string, payload any) error { _, err := audit.UpdateOne(ctx, bson.M{"source": source, "sourceId": id}, bson.M{"$set": bson.M{"source": source, "sourceId": id, "payload": payload, "archivedAt": nowISO()}}, options.Update().SetUpsert(true)) return err @@ -684,7 +684,7 @@ func (store *MongoStore) normalizeDefaultOrganizationRecord(ctx context.Context) } func (store *MongoStore) normalizeLegacyDefaultOrganizationReferences(ctx context.Context) error { - for _, collection := range []*mongo.Collection{ + for _, collection := range []mongoCollection{ store.projects, store.operators, store.builtinOperatorDeletions, @@ -816,7 +816,7 @@ func (store *MongoStore) cleanupOrphanedProjectResources(ctx context.Context) er if len(taskIDs) > 0 { taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}} - for _, col := range []*mongo.Collection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} { + for _, col := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} { if _, err := col.DeleteMany(ctx, taskFilter); err != nil { return err } @@ -824,7 +824,7 @@ func (store *MongoStore) cleanupOrphanedProjectResources(ctx context.Context) er } if len(itemIDs) > 0 { itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}} - for _, col := range []*mongo.Collection{store.humanStageAttempts, store.humanEvents} { + for _, col := range []mongoCollection{store.humanStageAttempts, store.humanEvents} { if _, err := col.DeleteMany(ctx, itemFilter); err != nil { return err } @@ -837,7 +837,7 @@ func (store *MongoStore) cleanupOrphanedProjectResources(ctx context.Context) er } for _, op := range []struct { - col *mongo.Collection + col mongoCollection filter bson.M }{ {store.annotationTasks, orphanFilter}, diff --git a/internal/store/object_sync.go b/internal/store/object_sync.go index f5181f07..65c9a764 100644 --- a/internal/store/object_sync.go +++ b/internal/store/object_sync.go @@ -837,7 +837,7 @@ func (store *MongoStore) ListObjectSyncSpaces(ctx context.Context, organizationI return decodeMany[domain.ObjectSyncSpaceRecord](ctx, cursor) } -func mongoObjectSyncGet[T any](ctx context.Context, collection *mongo.Collection, filter bson.M) (*T, error) { +func mongoObjectSyncGet[T any](ctx context.Context, collection mongoCollection, filter bson.M) (*T, error) { var item T err := collection.FindOne(ctx, filter).Decode(&item) if err == mongo.ErrNoDocuments { @@ -849,7 +849,7 @@ func mongoObjectSyncGet[T any](ctx context.Context, collection *mongo.Collection return &item, nil } -func mongoObjectSyncUpsert(ctx context.Context, collection *mongo.Collection, id string, item any) error { +func mongoObjectSyncUpsert(ctx context.Context, collection mongoCollection, id string, item any) error { _, err := collection.UpdateOne(ctx, bson.M{"id": id}, bson.M{"$set": item}, options.Update().SetUpsert(true)) return err } diff --git a/internal/store/platform_query_regression_test.go b/internal/store/platform_query_regression_test.go index 5e82a52d..56f06cee 100644 --- a/internal/store/platform_query_regression_test.go +++ b/internal/store/platform_query_regression_test.go @@ -12,6 +12,9 @@ import ( ) func TestPlatformClaimStageReadDoesNotFetchReviewBacklog(t *testing.T) { + // Exercise the operational/advisory path with real task scope; an orphan + // remains excluded even when expensive source audits are advisory. + t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory") var mu sync.Mutex var query bson.M ctx, s := poolLocalMongo(t, &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { @@ -30,12 +33,18 @@ func TestPlatformClaimStageReadDoesNotFetchReviewBacklog(t *testing.T) { t.Fatal(err) } docs := make([]any, 1042) + tasks := make([]any, 0, 42) for i := range docs { stage := domain.FanyuV58StageQualityReview if i < 42 { stage = domain.FanyuV58StageAnnotationCombined + tasks = append(tasks, bson.M{"id": fmt.Sprintf("task-%d", i), "organizationId": org, "projectId": project, "runId": "run"}) } - docs[i] = bson.M{"id": fmt.Sprintf("item-%d", i), "organizationId": org, "projectId": project, "pluginId": "annotation-qa", "batchId": batch.ID, "legacyTaskId": fmt.Sprintf("task-%d", i), "currentStageId": stage, "status": "ready", "assignedUserId": "", "supplierId": "supplier", "subject": bson.M{"itemKey": fmt.Sprintf("source-%d", i), "payload": bson.M{"large": strings.Repeat("x", 16<<10)}}} + docs[i] = bson.M{"id": fmt.Sprintf("item-%d", i), "organizationId": org, "projectId": project, "pluginId": "annotation-qa", "batchId": batch.ID, "runId": "run", "legacyTaskId": fmt.Sprintf("task-%d", i), "currentStageId": stage, "status": "ready", "assignedUserId": "", "supplierId": "supplier", "subject": bson.M{"itemKey": fmt.Sprintf("source-%d", i), "payload": bson.M{"large": strings.Repeat("x", 16<<10)}}} + } + docs = append(docs, bson.M{"id": "orphan", "organizationId": org, "projectId": project, "runId": "run", "pluginId": "annotation-qa", "batchId": batch.ID, "legacyTaskId": "missing", "currentStageId": domain.FanyuV58StageAnnotationCombined, "status": "ready", "assignedUserId": "", "supplierId": "supplier"}) + if _, err := s.annotationTasks.InsertMany(ctx, tasks); err != nil { + t.Fatal(err) } if _, err := s.humanItems.InsertMany(ctx, docs); err != nil { t.Fatal(err) diff --git a/internal/store/pool_search_backfill.go b/internal/store/pool_search_backfill.go index c73ac26d..8306ab88 100644 --- a/internal/store/pool_search_backfill.go +++ b/internal/store/pool_search_backfill.go @@ -26,7 +26,7 @@ func (s *MongoStore) BackfillHumanPoolSearchMetadata(ctx context.Context, org st func (s *MongoStore) backfillPoolSearchMetadata(ctx context.Context, org string, apply bool, collections []string) (PoolSearchBackfillResult, error) { result := PoolSearchBackfillResult{} for _, name := range collections { - collection := s.db.Collection(name) + collection := s.queryCollection(name) filter := bson.M{"organizationId": normalizedOrganizationID(org), "$expr": bson.M{"$not": bson.A{poolSearchNameIndexCurrentExpression(name)}}} cursor, err := collection.Find(ctx, filter, options.Find().SetBatchSize(16)) if err != nil { diff --git a/internal/store/private_derived_collections.go b/internal/store/private_derived_collections.go index a9375640..acb18255 100644 --- a/internal/store/private_derived_collections.go +++ b/internal/store/private_derived_collections.go @@ -5,5 +5,5 @@ import "droboticflow/internal/outputsnapshot" // Derived generations contain facts from multiple authorization scopes and // must only be served through their permission-aware reporting/search readers. func IsPrivateDerivedCollection(name string) bool { - return outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData" + return name == querySyncDeleteCollection || name == queryReplicaControl || outputsnapshot.IsPrivateCollection(name) || name == "queryReadModelControl" || name == "queryReadModelData" } diff --git a/internal/store/production_stats.go b/internal/store/production_stats.go index 8fe16fa9..09a7f302 100644 --- a/internal/store/production_stats.go +++ b/internal/store/production_stats.go @@ -42,14 +42,14 @@ func (s *MemoryStore) SaveProductionStatsScope(_ context.Context, org string, ne } func (s *MongoStore) GetProductionStatsScope(ctx context.Context, org string) (productionstats.Scope, error) { result := productionstats.Scope{ProjectIDs: []string{}} - err := s.db.Collection("productionStatsScopes").FindOne(ctx, bson.M{"_id": org}).Decode(&result) + err := s.queryCollection("productionStatsScopes").FindOne(ctx, bson.M{"_id": org}).Decode(&result) if err == mongo.ErrNoDocuments { err = nil } return result, err } func (s *MongoStore) SaveProductionStatsScope(ctx context.Context, org string, next productionstats.Scope, revision int) (bool, error) { - c := s.db.Collection("productionStatsScopes") + c := s.queryCollection("productionStatsScopes") if revision == 0 { _, err := c.InsertOne(ctx, bson.M{"_id": org, "projectIds": next.ProjectIDs, "revision": next.Revision, "updatedAt": next.UpdatedAt, "updatedBy": next.UpdatedBy}) if mongo.IsDuplicateKeyError(err) { @@ -122,23 +122,33 @@ func (s *MemoryStore) LoadProductionStats(ctx context.Context, org string, proje // lag by at most the TTL; the API-level response cache is flushed by every // non-GET /api request (e.g. scope edits) on top of that. func (s *MongoStore) LoadProductionStatsCached(ctx context.Context, org string, projects []string) (productionstats.Snapshot, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return productionstats.Snapshot{}, err + } else if replica != nil { + return replica.LoadProductionStatsCached(ctx, org, projects) + } keyProjects := append([]string(nil), projects...) sort.Strings(keyProjects) key := fmt.Sprintf("%p|%s|%s|%s", s.db.Client(), s.db.Name(), org, strings.Join(keyProjects, ",")) if snapshot, ok := productionStatsCache.get(key, dashboardStatsCacheTTL); ok { - return clone(snapshot), nil + return cloneProductionSnapshot(snapshot), nil } snapshot, err := s.LoadProductionStats(ctx, org, projects) if err != nil { return snapshot, err } - productionStatsCache.set(key, clone(snapshot), dashboardStatsCacheTTL) + productionStatsCache.set(key, cloneProductionSnapshot(snapshot), dashboardStatsCacheTTL) return snapshot, nil } // Project-scoped, unpaginated and projected reads: do not load video/artifact // payloads or inherit the automation list's 200-record UI limit. No migrations. func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projects []string) (productionstats.Snapshot, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return productionstats.Snapshot{}, err + } else if replica != nil { + return replica.LoadProductionStats(ctx, org, projects) + } ids := append([]string(nil), projects...) sort.Strings(ids) key, _ := json.Marshal([]any{org, ids}) @@ -154,7 +164,7 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec if r.Err != nil { return productionstats.Snapshot{}, r.Err } - return clone(r.Val.(productionstats.Snapshot)), nil + return cloneProductionSnapshot(r.Val.(productionstats.Snapshot)), nil } } @@ -166,7 +176,7 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec filter := organizationScopedFilter(org, bson.M{"projectId": bson.M{"$in": projects}}) hints := map[string]string{} - for collection, wanted := range map[*mongo.Collection]string{s.captureRecords: "production_capture_cover_v1", s.runs: productionRunsCoverIndex, s.humanEvents: productionEventFactsIndex} { + for collection, wanted := range map[mongoCollection]string{s.captureRecords: "production_capture_cover_v1", s.runs: productionRunsCoverIndex, s.humanEvents: productionEventFactsIndex} { indexes, e := collection.Indexes().List(ctx) if e != nil { var command mongo.CommandError @@ -193,7 +203,7 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec return d, e } } - read := func(c *mongo.Collection, f bson.M, projection bson.M, target any, preferred ...string) error { + read := func(c mongoCollection, f bson.M, projection bson.M, target any, preferred ...string) error { opts := options.Find().SetProjection(projection).SetMaxTime(15 * time.Second) hint := hints[c.Name()] // A project-leading covering index is unsuitable for legacy task-ID lookups. @@ -236,7 +246,7 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec } // Bound each related-key interval set before Mongo query planning. A single // OR containing every capture/run ID can take the entire request budget. - readRelated := func(c *mongo.Collection, key string, ids []string, projection bson.M, out any, extra bson.M) error { + readRelated := func(c mongoCollection, key string, ids []string, projection bson.M, out any, extra bson.M) error { unique := make([]string, 0, len(ids)) seen := map[string]bool{} for _, id := range ids { @@ -259,7 +269,7 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec } var g errgroup.Group g.Go(func() error { - projection := fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error") + projection := fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error", "queryExceptionCategory") if err := read(s.runs, filter, projection, &d.Runs, productionRunsCoverIndex); err != nil { return err } @@ -317,3 +327,11 @@ func (s *MongoStore) loadProductionStats(ctx context.Context, org string, projec return d, nil } + +func cloneProductionSnapshot(d productionstats.Snapshot) productionstats.Snapshot { + out := clone(d) + for i := range out.Runs { + out.Runs[i].QueryExceptionCategory = d.Runs[i].QueryExceptionCategory + } + return out +} diff --git a/internal/store/project_deletion.go b/internal/store/project_deletion.go index 59f53246..a9359234 100644 --- a/internal/store/project_deletion.go +++ b/internal/store/project_deletion.go @@ -131,8 +131,8 @@ func (s *MemoryStore) SaveProjectDeletion(_ context.Context, t domain.ProjectDel return true, nil } -func (s *MongoStore) deletionTasks() *mongo.Collection { - return s.db.Collection("project_deletion_tasks") +func (s *MongoStore) deletionTasks() mongoCollection { + return s.queryCollection("project_deletion_tasks") } func (s *MongoStore) MarkProjectDeleting(ctx context.Context, id, taskID string) error { r, e := s.projects.UpdateOne(ctx, bson.M{"id": id, "$or": bson.A{bson.M{"deletionTaskId": bson.M{"$exists": false}}, bson.M{"deletionTaskId": ""}, bson.M{"deletionTaskId": taskID}}}, bson.M{"$set": bson.M{"deletionTaskId": taskID}}) diff --git a/internal/store/query_collection_interface.go b/internal/store/query_collection_interface.go new file mode 100644 index 00000000..61e56cbc --- /dev/null +++ b/internal/store/query_collection_interface.go @@ -0,0 +1,36 @@ +package store + +import ( + "context" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Internal SDK seam: read helpers accept both ordinary collections and the +// source adapter that atomically tags writes. No business rule lives here. +type mongoCollection interface { + Name() string + Database() *mongo.Database + Indexes() mongo.IndexView + Find(context.Context, any, ...*options.FindOptions) (*mongo.Cursor, error) + FindOne(context.Context, any, ...*options.FindOneOptions) *mongo.SingleResult + Aggregate(context.Context, any, ...*options.AggregateOptions) (*mongo.Cursor, error) + InsertOne(context.Context, any, ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) + InsertMany(context.Context, []any, ...*options.InsertManyOptions) (*mongo.InsertManyResult, error) + UpdateOne(context.Context, any, any, ...*options.UpdateOptions) (*mongo.UpdateResult, error) + UpdateMany(context.Context, any, any, ...*options.UpdateOptions) (*mongo.UpdateResult, error) + ReplaceOne(context.Context, any, any, ...*options.ReplaceOptions) (*mongo.UpdateResult, error) + DeleteOne(context.Context, any, ...*options.DeleteOptions) (*mongo.DeleteResult, error) + DeleteMany(context.Context, any, ...*options.DeleteOptions) (*mongo.DeleteResult, error) + FindOneAndUpdate(context.Context, any, any, ...*options.FindOneAndUpdateOptions) *mongo.SingleResult + FindOneAndReplace(context.Context, any, any, ...*options.FindOneAndReplaceOptions) *mongo.SingleResult + FindOneAndDelete(context.Context, any, ...*options.FindOneAndDeleteOptions) *mongo.SingleResult + BulkWrite(context.Context, []mongo.WriteModel, ...*options.BulkWriteOptions) (*mongo.BulkWriteResult, error) + CountDocuments(context.Context, any, ...*options.CountOptions) (int64, error) + EstimatedDocumentCount(context.Context, ...*options.EstimatedDocumentCountOptions) (int64, error) + Distinct(context.Context, string, any, ...*options.DistinctOptions) ([]any, error) + Drop(context.Context) error +} + +var _ mongoCollection = (*mongo.Collection)(nil) +var _ mongoCollection = (*queryCollection)(nil) diff --git a/internal/store/query_read_builder.go b/internal/store/query_read_builder.go index ff6098b1..3e32edbe 100644 --- a/internal/store/query_read_builder.go +++ b/internal/store/query_read_builder.go @@ -16,7 +16,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes"} +var querySourceCollections = []string{"organizations", "projects", "runs", "captureRecords", "humanItems", "humanBatches", "annotationTasks", "annotationWorklogEvents", "humanEvents", "deliveryPackages", "annotationRevisions", "sliceRevisions", "annotationReviews", "users", "suppliers", "productionStatsScopes", "reviewOperations", "reworkBaselines", "reworkHandlings", "annotationIndexJobs"} // QueryReadBuilder runs only in a dedicated, explicitly configured process. // It uses the existing source loaders and publication protocol, never claims or diff --git a/internal/store/query_read_models.go b/internal/store/query_read_models.go index 18901b9f..79273f59 100644 --- a/internal/store/query_read_models.go +++ b/internal/store/query_read_models.go @@ -55,6 +55,12 @@ func (s *MongoStore) ConfigureQueryServing(cache queryserving.Cache, search *que s.queryServing = &queryServingOptions{cache: cache, search: search} } func (s *MongoStore) SharedQueryCache() queryserving.Cache { + if s.queryReplicaReads != nil { + if s.queryReplicaReads.target == nil { + return nil + } + return s.queryReplicaReads.target.SharedQueryCache() + } if s.queryServing == nil { return nil } @@ -86,10 +92,10 @@ func normalizedIDs(ids []string) []string { sort.Strings(out) return out } -func (s *MongoStore) queryControl() *mongo.Collection { - return s.db.Collection("queryReadModelControl") +func (s *MongoStore) queryControl() mongoCollection { + return s.queryCollection("queryReadModelControl") } -func (s *MongoStore) queryData() *mongo.Collection { return s.db.Collection("queryReadModelData") } +func (s *MongoStore) queryData() mongoCollection { return s.queryCollection("queryReadModelData") } func (s *MongoStore) acquireQueryModelLease(ctx context.Context) (queryModelLease, bool, error) { now := time.Now().UTC() owner := domain.NewID("query-builder") @@ -221,6 +227,11 @@ func (s *MongoStore) loadQueryModel(ctx context.Context, key string, h QueryMode // LoadProductionReport serves a derived, scope-qualified report if available. // Legacy reads remain the compatibility/failure path. No refresh is triggered. func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, projects []string) (productionstats.Report, string, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return productionstats.Report{}, "", err + } else if replica != nil { + return replica.LoadProductionReport(ctx, org, projects) + } if s.queryServing != nil { key := queryModelKey("production", org, projects) if head, ok := s.readQueryModelHead(ctx, key, dashboardStatsCacheTTL); ok { @@ -239,6 +250,11 @@ func (s *MongoStore) LoadProductionReport(ctx context.Context, org string, proje } func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, projects []string, query string) ([]byte, bool) { + if replica, err := s.reportReplica(ctx); err != nil { + return nil, false + } else if replica != nil { + return replica.PreparedProductionResponse(ctx, org, projects, query) + } if s.queryServing == nil { return nil, false } @@ -253,6 +269,12 @@ func (s *MongoStore) PreparedProductionResponse(ctx context.Context, org string, return b, hit } func (s *MongoStore) RememberProductionResponse(ctx context.Context, org string, projects []string, query, asOf string, value any) { + if replica, err := s.reportReplica(ctx); err != nil { + return + } else if replica != nil { + replica.RememberProductionResponse(ctx, org, projects, query, asOf, value) + return + } if s.queryServing == nil || s.SharedQueryCache() == nil { return } diff --git a/internal/store/query_replica.go b/internal/store/query_replica.go new file mode 100644 index 00000000..21206e11 --- /dev/null +++ b/internal/store/query_replica.go @@ -0,0 +1,371 @@ +package store + +import ( + "context" + "errors" + "sort" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/writeconcern" +) + +const queryReplicaControl = "queryReplicaControl" +const queryReplicaSchema = "query-replica/v1" + +var ErrQueryReplicaLease = errors.New("query replica lease lost") +var ErrQueryReplicaNotReady = errors.New("查询暂不可用,请稍后重试") + +// QueryReplica copies only the explicit query-fact schema. The source does not +// require transactions or Change Streams. Checkpoints and target writes share +// a target transaction; source acknowledgements happen only after its commit. +// Tracking must cover every source writer before Bootstrap is started. +type QueryReplica struct { + Source, Target *MongoStore + owner string + epoch int64 + BatchSize int + dirtyAfter map[string]any + reconcileNext int +} +type queryReplicaCheckpoint struct { + ID string `bson:"_id"` + Last any `bson:"last,omitempty"` + HasLast bool `bson:"hasLast"` + Complete bool `bson:"complete"` + CheckedAt time.Time `bson:"checkedAt"` +} + +func NewQueryReplica(source, target *MongoStore) (*QueryReplica, error) { + if source == nil || target == nil || source.db == nil || target.db == nil || source.db.Client() == target.db.Client() { + return nil, errors.New("separate source and query clients required") + } + target.queryReplicaFactStore = true + return &QueryReplica{Source: source, Target: target, owner: domain.NewID("query-projector"), BatchSize: 200, dirtyAfter: map[string]any{}}, nil +} +func (q *QueryReplica) control() *mongo.Collection { + return q.Target.db.Collection(queryReplicaControl) +} +func (q *QueryReplica) limit() int64 { + if q.BatchSize < 1 || q.BatchSize > 1000 { + return 200 + } + return int64(q.BatchSize) +} +func replicaCollections() []string { + names := make([]string, 0, len(queryReplicaFields)) + for n := range queryReplicaFields { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// VerifyTopology rejects two database names on the same Mongo deployment. A +// replica set is required only on the independent target, for fencing/CDC. +func (q *QueryReplica) VerifyTopology(ctx context.Context) error { + var a, b bson.M + if e := q.Source.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&a); e != nil { + return e + } + if e := q.Target.db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&b); e != nil { + return e + } + if b["setName"] == nil { + return errors.New("query database must support transactions") + } + atop, btop := deletionDoc(a["topologyVersion"]), deletionDoc(b["topologyVersion"]) + if atop["processId"] != nil && atop["processId"] == btop["processId"] { + return errors.New("query database shares source process") + } + if a["setName"] != nil && a["setName"] == b["setName"] { + return errors.New("query database shares source replica set") + } + return nil +} + +// EnsureSourceIndexes is an explicit administrative operation, never startup +// behavior. Production execution requires a separately approved index plan. +func (q *QueryReplica) EnsureSourceIndexes(ctx context.Context) error { + for _, n := range replicaCollections() { + _, e := q.Source.db.Collection(n).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: querySyncField + ".dirty", Value: 1}, {Key: "_id", Value: 1}}, Options: options.Index().SetName("query_sync_dirty_v1").SetPartialFilterExpression(bson.M{querySyncField + ".dirty": true})}) + if e != nil { + return e + } + _, e = q.Source.db.Collection(n).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: querySyncField + ".deleteClaim", Value: 1}}, Options: options.Index().SetName("query_sync_delete_claim_v1").SetSparse(true)}) + if e != nil { + return e + } + } + _, e := q.Source.db.Collection(querySyncDeleteCollection).Indexes().CreateMany(ctx, []mongo.IndexModel{ + {Keys: bson.D{{Key: "phase", Value: 1}, {Key: "leaseUntil", Value: 1}}, Options: options.Index().SetName("query_sync_delete_v1")}, + {Keys: bson.D{{Key: "collection", Value: 1}, {Key: "phase", Value: 1}, {Key: "audit", Value: 1}}, Options: options.Index().SetName("query_sync_delete_pending_v1")}, + {Keys: bson.D{{Key: "expireAt", Value: 1}}, Options: options.Index().SetName("query_sync_delete_retention_v1").SetExpireAfterSeconds(0)}, + }) + return e +} +func (q *QueryReplica) Acquire(ctx context.Context) error { + if e := q.bind(ctx); e != nil { + return e + } + now := time.Now().UTC() + _, e := q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$setOnInsert": bson.M{"owner": "", "epoch": int64(0), "until": time.Time{}}}, options.Update().SetUpsert(true)) + if e != nil && !mongo.IsDuplicateKeyError(e) { + return e + } + var d struct { + Epoch int64 `bson:"epoch"` + } + e = q.control().FindOneAndUpdate(ctx, bson.M{"_id": "lease", "$or": bson.A{bson.M{"until": bson.M{"$lte": now}}, bson.M{"owner": q.owner}}}, bson.M{"$set": bson.M{"owner": q.owner, "until": now.Add(30 * time.Second)}, "$inc": bson.M{"epoch": 1}}, options.FindOneAndUpdate().SetReturnDocument(options.After)).Decode(&d) + if errors.Is(e, mongo.ErrNoDocuments) { + return ErrQueryReplicaLease + } + q.epoch = d.Epoch + return e +} +func (q *QueryReplica) transaction(ctx context.Context, fn func(mongo.SessionContext) error) error { + session, e := q.Target.db.Client().StartSession() + if e != nil { + return e + } + defer session.EndSession(ctx) + _, e = session.WithTransaction(ctx, func(sc mongo.SessionContext) (any, error) { + now := time.Now().UTC() + r, e := q.control().UpdateOne(sc, bson.M{"_id": "lease", "owner": q.owner, "epoch": q.epoch, "until": bson.M{"$gt": now}}, bson.M{"$set": bson.M{"until": now.Add(30 * time.Second)}}) + if e != nil { + return nil, e + } + if r.MatchedCount != 1 { + return nil, ErrQueryReplicaLease + } + return nil, fn(sc) + }, options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.Majority())) + return e +} +func (q *QueryReplica) checkpoint(ctx context.Context, n string) (queryReplicaCheckpoint, error) { + var cp queryReplicaCheckpoint + e := q.control().FindOne(ctx, bson.M{"_id": "bootstrap/" + n}).Decode(&cp) + if errors.Is(e, mongo.ErrNoDocuments) { + return queryReplicaCheckpoint{ID: "bootstrap/" + n}, nil + } + return cp, e +} +func (q *QueryReplica) apply(ctx mongo.SessionContext, n string, records []queryReplicaRecord) error { + if len(records) == 0 { + return nil + } + c := q.Target.db.Collection(n) + ids := make([]any, 0, len(records)) + seen := map[string]bool{} + for _, r := range records { + key := querySourceIdentity(n, r.Document["_id"]) + if seen[key] { + return ErrQueryReplicaUnsupported + } + seen[key] = true + ids = append(ids, r.Document["_id"]) + } + cur, e := c.Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_queryReplica": 1})) + if e != nil { + return e + } + current := map[string]bson.M{} + for cur.Next(ctx) { + var d bson.M + if e = cur.Decode(&d); e != nil { + cur.Close(ctx) + return e + } + current[querySourceIdentity(n, d["_id"])] = deletionDoc(d["_queryReplica"]) + } + e = cur.Err() + cur.Close(ctx) + if e != nil { + return e + } + models := make([]mongo.WriteModel, 0, len(records)) + for i, r := range records { + m := current[querySourceIdentity(n, r.Document["_id"])] + records[i].TargetSourceID = m["sourceId"] + records[i].TargetRevision = int64(supplierNumber(m["revision"])) + if bsonEqual(m["sourceId"], r.SourceID) && m["schema"] == queryReplicaSchema && (records[i].TargetRevision > r.Revision || (records[i].TargetRevision == r.Revision && m["digest"] == deletionDoc(r.Document["_queryReplica"])["digest"])) { + continue + } + records[i].TargetSourceID = r.SourceID + records[i].TargetRevision = r.Revision + models = append(models, mongo.NewReplaceOneModel().SetFilter(bson.M{"_id": r.Document["_id"]}).SetReplacement(r.Document).SetUpsert(true)) + } + if len(models) == 0 { + return nil + } + _, e = c.BulkWrite(ctx, models, options.BulkWrite().SetOrdered(true)) + if e != nil { + return e + } + _, e = q.control().UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$inc": bson.M{"factEpoch": int64(1), "collectionEpoch." + n: int64(1)}}, options.Update().SetUpsert(true)) + return e +} + +func bsonEqual(a, b any) bool { + x, e := bson.Marshal(bson.M{"v": a}) + if e != nil { + return false + } + y, e := bson.Marshal(bson.M{"v": b}) + return e == nil && string(x) == string(y) +} +func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]queryReplicaRecord, any, error) { + opts := options.Find().SetProjection(querySourceProjection(n)).SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(q.limit()).SetBatchSize(int32(q.limit())) + if filter[querySyncField+".dirty"] == true { + opts.SetHint("query_sync_dirty_v1") + } else { + opts.SetHint("_id_") + } + cur, e := q.Source.db.Collection(n).Find(ctx, filter, opts) + if e != nil { + return nil, nil, e + } + defer cur.Close(ctx) + records := []queryReplicaRecord{} + var last any + for cur.Next(ctx) { + var d bson.M + if e = cur.Decode(&d); e != nil { + return nil, nil, e + } + last = d["_id"] + r, e := queryExportRecord(n, d) + if e != nil { + return nil, nil, e + } + records = append(records, r) + } + return records, last, cur.Err() +} +func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []queryReplicaRecord) error { + models := make([]mongo.WriteModel, 0, len(records)) + for _, r := range records { + if bsonEqual(r.TargetSourceID, r.SourceID) && r.TargetRevision > r.Revision { + // An old writer removed/restored an earlier marker. Never acknowledge + // that regression as current: re-mark metadata above the durable sink rev. + models = append(models, mongo.NewUpdateOneModel().SetFilter(queryReplicaSourceCAS(r)).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(r.TargetRevision + 1)})) + continue + } + if r.Revision > 0 { + models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": r.SourceID, querySyncField + ".revision": r.Revision, querySyncField + ".dirty": true}).SetUpdate(bson.M{"$set": bson.M{querySyncField + ".dirty": false}})) + } + } + if len(models) == 0 { + return nil + } + _, e := q.Source.db.Collection(n).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)) + return e +} + +// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across +// mixed historical _id types; a plain $gt would silently skip other BSON types. +// This one-time pass is rate limited by the caller, never a per-request scan. +func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int, error) { + if queryReplicaFields[n] == "" { + return false, 0, ErrQueryReplicaUnsupported + } + cp, e := q.checkpoint(ctx, n) + if e != nil || cp.Complete { + return cp.Complete, 0, e + } + filter := bson.M{} + if cp.HasLast || cp.Last != nil { + filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": cp.Last}}} + } + rows, last, e := q.read(ctx, n, filter) + if e != nil { + return false, 0, e + } + done := int64(len(rows)) < q.limit() + e = q.transaction(ctx, func(sc mongo.SessionContext) error { + if e := q.apply(sc, n, rows); e != nil { + return e + } + update := bson.M{"complete": done} + if len(rows) > 0 { + update["hasLast"] = true + update["last"] = last + } + _, e := q.control().UpdateOne(sc, bson.M{"_id": cp.ID}, bson.M{"$set": update}, options.Update().SetUpsert(true)) + return e + }) + if e != nil { + return false, 0, e + } + return done, len(rows), q.acknowledge(ctx, n, rows) +} + +// DrainStep never acknowledges a newer concurrent save. A crash after target +// commit simply replays the same version; a crash before commit retains dirty. +func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) { + if queryReplicaFields[n] == "" { + return 0, ErrQueryReplicaUnsupported + } + filter := bson.M{querySyncField + ".dirty": true} + if last := q.dirtyAfter[n]; last != nil { + filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": last}}} + } + rows, last, e := q.read(ctx, n, filter) + if e == nil { + if int64(len(rows)) < q.limit() { + delete(q.dirtyAfter, n) + } else { + q.dirtyAfter[n] = last + } + } + if e != nil { + return 0, e + } + e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) }) + if e != nil { + return 0, e + } + return len(rows), q.acknowledge(ctx, n, rows) +} + +// EnableQuerySourceTracking is deliberately explicit and defaults off. Every +// writer role must be upgraded before a replica may be marked ready. +func (s *MongoStore) EnableQuerySourceTracking(enabled bool) { s.querySyncEnabled.Store(enabled) } + +// RunOnce performs at most one batch per collection. Backfill is resumable and +// bounded; the process does not begin unbounded scans on an HTTP cache miss. +func (q *QueryReplica) RunOnce(ctx context.Context, bootstrap bool) (bool, error) { + for _, n := range replicaCollections() { + cp, e := q.checkpoint(ctx, n) + if e != nil { + return false, e + } + if !cp.Complete { + if !bootstrap { + return false, ErrQueryReplicaNotReady + } + if _, _, e = q.BootstrapStep(ctx, n); e != nil { + return false, e + } + } + if _, e = q.DrainStep(ctx, n); e != nil { + return false, e + } + } + if _, e := q.ReconcileOneDue(ctx, 6*time.Hour); e != nil { + return false, e + } + if _, e := q.ReconcileDeleteStep(ctx); e != nil { + return false, e + } + if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady { + return false, nil + } else if e != nil { + return false, e + } + return true, nil +} diff --git a/internal/store/query_replica_barrier.go b/internal/store/query_replica_barrier.go new file mode 100644 index 00000000..2866c32f --- /dev/null +++ b/internal/store/query_replica_barrier.go @@ -0,0 +1,221 @@ +package store + +import ( + "context" + "sync" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const queryReplicaPendingLimit = 1000 + +type queryReplicaPoolBarrier struct { + batchEpochOnce sync.Once + batchEpoch int64 + batchEpochErr error + once sync.Once + org string + err error + items []domain.HumanItemRecord + ids map[string]bool + batches map[string]bool + counts bool +} + +func WithQueryReplicaCountRead(ctx context.Context) context.Context { + if b, ok := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier); ok && b.counts { + return ctx + } + return context.WithValue(ctx, queryReplicaPoolReadKey{}, &queryReplicaPoolBarrier{counts: true}) +} +func (s *MongoStore) countReplicaBarrier(ctx context.Context, org string) (*queryReplicaPoolBarrier, error) { + b, ok := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier) + if !ok || !b.counts { + return nil, nil + } + b.once.Do(func() { + b.org = org + b.ids = map[string]bool{} + b.batches = map[string]bool{} + // Acknowledgement follows the majority-committed target transaction. Thus a + // clean source record is already present in the query DB; pending records + // are overlaid from authority. Never assume a dirty record is absent. + for _, name := range []string{"humanItems", "humanBatches"} { + projection := bson.M{"id": 1} + if name == "humanItems" { + projection = humanItemListProjection(domain.HumanItemQuery{PoolCandidateProjection: true}) + } + cur, e := s.db.Collection(name).Find(ctx, organizationScopedFilter(org, bson.M{querySyncField + ".dirty": true}), options.Find().SetProjection(projection).SetHint("query_sync_dirty_v1").SetLimit(queryReplicaPendingLimit+1)) + if e != nil { + b.err = e + return + } + n := 0 + for cur.Next(ctx) { + n++ + if name == "humanItems" { + var item domain.HumanItemRecord + if e = cur.Decode(&item); e != nil { + break + } + b.items = append(b.items, item) + b.ids[item.ID] = true + } else { + var row struct { + ID string `bson:"id"` + } + if e = cur.Decode(&row); e != nil { + break + } + b.batches[row.ID] = true + } + } + if e == nil { + e = cur.Err() + } + cur.Close(ctx) + if e != nil { + b.err = e + return + } + if n > queryReplicaPendingLimit { + b.err = ErrQueryReplicaNotReady + return + } + } + // Deletion has no surviving dirty document. Wait for the durable identity + // sweep rather than serving stale task/batch scopes or rescanning all data. + e := s.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"audit": bson.M{"$ne": true}, "collection": bson.M{"$in": bson.A{"humanItems", "humanBatches", "projects"}}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err() + if e == nil { + b.err = ErrQueryReplicaNotReady + } else if e != mongo.ErrNoDocuments { + b.err = e + } + }) + if b.org != org { + return nil, ErrQueryReplicaNotReady + } + return b, b.err +} +func (s *MongoStore) visitReplicaPoolCounts(ctx context.Context, target *MongoStore, org string, q domain.HumanItemQuery, visit func([]domain.HumanItemRecord) error) error { + b, e := s.countReplicaBarrier(ctx, org) + if e != nil { + return e + } + if b == nil { + return target.VisitHumanPoolCountCandidates(ctx, org, q, visit) + } + e = target.VisitHumanPoolCountCandidates(ctx, org, q, func(rows []domain.HumanItemRecord) error { + kept := rows[:0] + for _, r := range rows { + if !b.ids[r.ID] { + kept = append(kept, r) + } + } + if len(kept) == 0 { + return nil + } + return visit(kept) + }) + if e != nil { + return e + } + return s.visitReplicaPendingCounts(ctx, org, q, b, visit) +} + +// Match only the bounded dirty IDs against authority. Ordinary saves must not +// disable filtered or blind counts while the asynchronous projector catches up. +// Reuse the exact legacy matcher without its broad discovery query. +func (s *MongoStore) visitReplicaPendingCounts(ctx context.Context, org string, q domain.HumanItemQuery, b *queryReplicaPoolBarrier, visit func([]domain.HumanItemRecord) error) error { + if len(b.items) == 0 { + return nil + } + batchByID := map[string]domain.HumanBatchRecord{} + if q.BlindAnnotationActorID != "" { + ids := []string{} + seen := map[string]bool{} + for _, item := range b.items { + if item.BatchID != "" && !seen[item.BatchID] { + ids = append(ids, item.BatchID) + seen[item.BatchID] = true + } + } + if len(ids) > 0 { + rows, err := s.ListHumanBatchAccessRecords(ctx, org, ids) + if err != nil { + return err + } + for _, batch := range rows { + batchByID[batch.ID] = batch + } + } + } + pending := []domain.HumanItemRecord{} + appendVisible := func(item domain.HumanItemRecord) { + if humanItemMatches(item, q) && blindAnnotationVisible(item, batchByID[item.BatchID], q.BlindAnnotationActorID) { + pending = append(pending, humanItemListProjectionClone(item, humanItemListPayloadKeysRemoved)) + } + } + if humanPoolFiltered(q) { + ids := make([]string, 0, len(b.ids)) + for id := range b.ids { + ids = append(ids, id) + } + filter := bson.M{"$and": bson.A{humanItemMongoFilter(org, q), bson.M{"id": bson.M{"$in": ids}}}} + if err := s.visitJoinedHumanPoolSearchRows(ctx, org, q, filter, true, func(row adminPoolRow) (bool, error) { + appendVisible(row.HumanItemRecord) + return false, nil + }); err != nil { + return err + } + } else { + for _, item := range b.items { + appendVisible(item) + } + } + if len(pending) > 0 { + return visit(pending) + } + return nil +} + +func (s *MongoStore) replicaBatchAccess(ctx context.Context, target *MongoStore, org string, ids []string) ([]domain.HumanBatchRecord, error) { + b, e := s.countReplicaBarrier(ctx, org) + if e != nil { + return nil, e + } + if b == nil { + return nil, ErrQueryReplicaNotReady + } + clean, dirty := []string{}, []string{} + for _, id := range ids { + if b.batches[id] { + dirty = append(dirty, id) + } else { + clean = append(clean, id) + } + } + rows, e := target.ListHumanBatchAccessRecords(ctx, org, clean) + if e != nil { + return nil, e + } + if len(dirty) > 0 { + fresh, e := s.listAuthoritativeBatchAccess(ctx, org, dirty) + if e != nil { + return nil, e + } + rows = append(rows, fresh...) + } + return rows, nil +} + +func queryDeletePendingPhases() bson.A { + return bson.A{"prepared", "deleting", "complete", "cancelled", "reconcile"} +} +func (s *MongoStore) UsesQueryReplicaCountRead(ctx context.Context) bool { + b, ok := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier) + return ok && b.counts && s.UsesQueryReplicaPool(ctx) +} diff --git a/internal/store/query_replica_batch_cache.go b/internal/store/query_replica_batch_cache.go new file mode 100644 index 00000000..8a7780be --- /dev/null +++ b/internal/store/query_replica_batch_cache.go @@ -0,0 +1,102 @@ +package store + +import ( + "context" + "sync" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type queryBatchFacts struct { + once sync.Once + permit chan struct{} + mu sync.Mutex + epoch int64 + rows map[string]*domain.HumanBatchRecord +} + +func cloneReplicaBatch(v domain.HumanBatchRecord) domain.HumanBatchRecord { + v.Stages = append([]domain.HumanStageDefinition(nil), v.Stages...) + for i := range v.Stages { + v.Stages[i].RejectTargets = append([]string(nil), v.Stages[i].RejectTargets...) + } + return v +} + +// Only clean, majority-committed query facts use this cache. Source pending +// policy rows bypass it in replicaBatchAccess; account/grant checks never enter +// it. Unknown IDs are negative-cached only at this collection's fact epoch. +func (s *MongoStore) cachedReplicaBatchAccess(ctx context.Context, org string, ids []string) ([]domain.HumanBatchRecord, error) { + b, ok := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier) + if !ok || !b.counts { + return s.readBatchAccess(ctx, org, ids) + } + b.batchEpochOnce.Do(func() { + var st struct { + Epochs map[string]int64 `bson:"collectionEpoch"` + } + b.batchEpochErr = s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}, options.FindOne().SetProjection(bson.M{"collectionEpoch.humanBatches": 1})).Decode(&st) + b.batchEpoch = st.Epochs["humanBatches"] + }) + if b.batchEpochErr != nil { + return nil, b.batchEpochErr + } + c := &s.queryReplicaBatchFacts + c.once.Do(func() { + c.permit = make(chan struct{}, 1) + c.permit <- struct{}{} + c.rows = map[string]*domain.HumanBatchRecord{} + }) + // One cold batch fill per process, cancelable while waiting. Shared facts + // avoid decoding the same 26k batch policies separately for every actor. + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-c.permit: + } + defer func() { c.permit <- struct{}{} }() + c.mu.Lock() + if b.batchEpoch > c.epoch { + clear(c.rows) + c.epoch = b.batchEpoch + } + missing := []string{} + seen := map[string]bool{} + for _, id := range ids { + key := org + ":" + id + if _, ok := c.rows[key]; !ok && !seen[id] { + missing = append(missing, id) + seen[id] = true + } + } + c.mu.Unlock() + if len(missing) > 0 { + rows, e := s.readBatchAccess(ctx, org, missing) + if e != nil { + return nil, e + } + c.mu.Lock() + if len(c.rows)+len(missing) > 32768 { + clear(c.rows) + } + for _, id := range missing { + c.rows[org+":"+id] = nil + } + for _, v := range rows { + r := cloneReplicaBatch(v) + c.rows[org+":"+v.ID] = &r + } + c.mu.Unlock() + } + out := make([]domain.HumanBatchRecord, 0, len(ids)) + c.mu.Lock() + for _, id := range ids { + if r := c.rows[org+":"+id]; r != nil { + out = append(out, cloneReplicaBatch(*r)) + } + } + c.mu.Unlock() + return out, nil +} diff --git a/internal/store/query_replica_binding.go b/internal/store/query_replica_binding.go new file mode 100644 index 00000000..ad1ff10c --- /dev/null +++ b/internal/store/query_replica_binding.go @@ -0,0 +1,55 @@ +package store + +import ( + "context" + "os" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Namespace is an explicit deployment identity, not a credential or a process +// ID. Local fixtures without environment configuration use their isolated DB. +func queryReplicaNamespace(sourceDatabase string) string { + if ns := os.Getenv("QUERY_SERVING_NAMESPACE"); ns != "" { + return ns + } + return "local:" + sourceDatabase +} + +type queryReplicaBinding struct { + SourceDatabase string `bson:"sourceDatabase"` + Namespace string `bson:"namespace"` +} + +func (q *QueryReplica) binding() queryReplicaBinding { + return queryReplicaBinding{q.Source.db.Name(), queryReplicaNamespace(q.Source.db.Name())} +} +func (q *QueryReplica) bind(ctx context.Context) error { + want := q.binding() + var got queryReplicaBinding + e := q.control().FindOneAndUpdate(ctx, bson.M{"_id": "source-binding"}, bson.M{"$setOnInsert": want}, options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After)).Decode(&got) + if e != nil { + return e + } + if got != want { + return ErrQueryReplicaNotReady + } + return nil +} +func (s *MongoStore) verifyReplicaBinding(target *MongoStore) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + return s.verifyReplicaBindingContext(ctx, target) +} +func (s *MongoStore) verifyReplicaBindingContext(ctx context.Context, target *MongoStore) error { + var got queryReplicaBinding + if e := target.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "source-binding"}).Decode(&got); e != nil { + return ErrQueryReplicaNotReady + } + if got.SourceDatabase != s.db.Name() || got.Namespace != queryReplicaNamespace(s.db.Name()) { + return ErrQueryReplicaNotReady + } + return nil +} diff --git a/internal/store/query_replica_config.go b/internal/store/query_replica_config.go new file mode 100644 index 00000000..9601658e --- /dev/null +++ b/internal/store/query_replica_config.go @@ -0,0 +1,54 @@ +package store + +import ( + "context" + "errors" + "os" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readconcern" + "go.mongodb.org/mongo-driver/mongo/readpref" +) + +// ConfigureQueryReplicaFromEnvironment is an entry-point operation. Invalid or +// unavailable reporting infrastructure cannot disable existing work mutations. +// Explicit serving mode fails closed for reports, with no source-scan fallback. +func (s *MongoStore) ConfigureQueryReplicaFromEnvironment(ctx context.Context) (func(), error) { + if os.Getenv("QUERY_REPLICA_REPORTS") != "1" { + return func() {}, nil + } + s.queryReplicaReads = &queryReplicaReads{pool: os.Getenv("QUERY_REPLICA_POOLS") == "1"} + if os.Getenv("QUERY_SOURCE_TRACKING") != "1" { + return func() {}, errors.New("query reads require source writer tracking") + } + uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB") + if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" { + return func() {}, errors.New("explicit query database and namespace required") + } + c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority()).SetMaxPoolSize(20).SetServerSelectionTimeout(2*time.Second)) + if e != nil { + return func() {}, errors.New("query database connection unavailable") + } + cleanup := func() { + short, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = c.Disconnect(short) + } + target := NewMongoStore(c.Database(db)) + closeCache, e := target.ConfigureQueryServingFromEnvironment() + if e != nil { + cleanup() + return func() {}, errors.New("query accelerator configuration invalid") + } + r, e := newQueryReplicaReads(s, target, 30*time.Second) + if e != nil { + closeCache() + cleanup() + return func() {}, e + } + s.queryReplicaReads = r + s.EnableQueryReplicaPoolReads(os.Getenv("QUERY_REPLICA_POOLS") == "1") + return func() { closeCache(); cleanup() }, nil +} diff --git a/internal/store/query_replica_count_cache.go b/internal/store/query_replica_count_cache.go new file mode 100644 index 00000000..7f9f5a47 --- /dev/null +++ b/internal/store/query_replica_count_cache.go @@ -0,0 +1,255 @@ +package store + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "maps" + "sync" + "time" + "unsafe" + + "droboticflow/internal/domain" + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const queryCountMemoryBudget = 64 << 20 +const queryCountTTL = 2 * time.Second + +var errQueryCountCacheCapacity = errors.New("count candidate cache capacity exceeded") + +type queryCountEntry struct { + rows []domain.HumanItemRecord + expires time.Time + bytes int +} +type queryCountFlight struct { + ctx context.Context + cancel context.CancelFunc + done chan struct{} + waiters int + rows []domain.HumanItemRecord + err error +} +type queryCountCache struct { + mu sync.Mutex + entries map[string]queryCountEntry + flights map[string]*queryCountFlight + bytes int +} + +func cloneQueryCountRow(v domain.HumanItemRecord) domain.HumanItemRecord { + v.Subject.Payload = nil + v.PoolSearch = nil + v.CanClaim = nil + v.TraceSummary = nil + v.ReviewHistory = nil + v.LegacyAssignment = maps.Clone(v.LegacyAssignment) + if v.AcceptanceClaim != nil { + r := *v.AcceptanceClaim + v.AcceptanceClaim = &r + } + if v.AccountHandoff != nil { + r := *v.AccountHandoff + v.AccountHandoff = &r + } + if v.ProcessingJob != nil { + r := *v.ProcessingJob + r.Input = nil + v.ProcessingJob = &r + } + return v +} +func (s *MongoStore) countFactEpoch(ctx context.Context) (int64, error) { + var st struct { + Epoch int64 `bson:"factEpoch"` + } + e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}, options.FindOne().SetProjection(bson.M{"factEpoch": 1})).Decode(&st) + return st.Epoch, e +} +func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, bytes int) { + if bytes > queryCountMemoryBudget { + return + } + now := time.Now() + for k, e := range c.entries { + if !now.Before(e.expires) { + delete(c.entries, k) + c.bytes -= e.bytes + } + } + if len(c.entries) >= 32 || c.bytes+bytes > queryCountMemoryBudget { + clear(c.entries) + c.bytes = 0 + } + c.entries[key] = queryCountEntry{rows: rows, bytes: bytes, expires: now.Add(queryCountTTL)} + c.bytes += bytes +} + +// The cache stores query candidates, NEVER authorization or a final count. +// The API still evaluates current permissions on private per-request copies. +// The target fact epoch is committed atomically with fact changes, so a cold +// miss or invalidation reads the query DB, never the authority DB. +func (s *MongoStore) visitCachedReplicaCounts(ctx context.Context, org string, q domain.HumanItemQuery, visit func([]domain.HumanItemRecord) error) error { + epoch, e := s.countFactEpoch(ctx) + if e != nil { + return e + } + q.Limit = 0 + q.Cursor = nil + key := queryserving.Key("replica-count-candidates/v1", s.db.Name(), org, epoch, q) + c := &s.queryReplicaCounts + c.mu.Lock() + if c.entries == nil { + c.entries = map[string]queryCountEntry{} + c.flights = map[string]*queryCountFlight{} + } + var rows []domain.HumanItemRecord + if entry, ok := c.entries[key]; ok && time.Now().Before(entry.expires) { + rows = entry.rows + c.mu.Unlock() + } else { + f := c.flights[key] + if f == nil && len(c.flights) >= 16 { + c.mu.Unlock() + return s.visitUncachedHumanPoolCounts(ctx, org, q, visit) + } + if f == nil || f.ctx.Err() != nil { + work, cancel := context.WithTimeout(context.Background(), 8*time.Second) + f = &queryCountFlight{ctx: work, cancel: cancel, done: make(chan struct{})} + c.flights[key] = f + go func() { + result, bytes, err := s.loadReplicaCountRows(work, org, q, key) + cacheable := true + if err == nil { + next, errEpoch := s.countFactEpoch(work) + if errEpoch != nil { + err = errEpoch + } else if next != epoch { + cacheable = false + } + } + c.mu.Lock() + f.rows, f.err = result, err + if c.flights[key] == f { + delete(c.flights, key) + if err == nil && cacheable && work.Err() == nil { + c.remember(key, result, bytes) + } + } + close(f.done) + c.mu.Unlock() + cancel() + }() + } + f.waiters++ + c.mu.Unlock() + select { + case <-ctx.Done(): + e = ctx.Err() + case <-f.done: + rows, e = f.rows, f.err + } + c.mu.Lock() + f.waiters-- + if f.waiters == 0 { + f.cancel() + if c.flights[key] == f { + delete(c.flights, key) + } + } + c.mu.Unlock() + if errors.Is(e, errQueryCountCacheCapacity) { + return s.visitUncachedHumanPoolCounts(ctx, org, q, visit) + } + if e != nil { + return e + } + } + buffer := make([]domain.HumanItemRecord, min(len(rows), humanPoolCountBatchSize)) + for start := 0; start < len(rows); start += len(buffer) { + if e := ctx.Err(); e != nil { + return e + } + end := min(start+len(buffer), len(rows)) + page := buffer[:end-start] + for i := range page { + page[i] = cloneQueryCountRow(rows[start+i]) + } + if e := visit(page); e != nil { + return e + } + } + return ctx.Err() +} +func (s *MongoStore) loadReplicaCountRows(ctx context.Context, org string, q domain.HumanItemQuery, key string) ([]domain.HumanItemRecord, int, error) { + // BSON retains private eligibility metadata that JSON intentionally omits. + if cache := s.SharedQueryCache(); cache != nil { + if raw, ok := cache.Get(ctx, key); ok { + if rows, e := decodeQueryCountRows(raw); e == nil { + return rows, queryCountRowsBytes(rows), nil + } + } + } + rows := []domain.HumanItemRecord{} + e := s.visitUncachedHumanPoolCounts(ctx, org, q, func(page []domain.HumanItemRecord) error { + if len(rows)+len(page) > 50000 { + return errQueryCountCacheCapacity + } + for _, row := range page { + rows = append(rows, cloneQueryCountRow(row)) + } + return nil + }) + if e != nil { + return nil, 0, e + } + if cache := s.SharedQueryCache(); cache != nil { + if raw, e := encodeQueryCountRows(rows); e == nil && len(raw) <= queryserving.MaxValueBytes { + cache.Put(ctx, key, raw, queryCountTTL) + } + } + return rows, queryCountRowsBytes(rows), nil +} +func queryCountRowsBytes(rows []domain.HumanItemRecord) int { + n := len(rows) * int(unsafe.Sizeof(domain.HumanItemRecord{})) + for _, r := range rows { + n += len(r.ID) + len(r.ProjectID) + len(r.LegacyTaskID) + len(r.BatchID) + len(r.AssignedUserID) + len(r.LatestAuthorUserID) + 512 + } + return n +} +func encodeQueryCountRows(rows []domain.HumanItemRecord) ([]byte, error) { + raw, e := bson.Marshal(bson.M{"rows": rows}) + if e != nil || len(raw) > 128<<20 { + return nil, errors.New("count cache exceeds budget") + } + var b bytes.Buffer + w := gzip.NewWriter(&b) + if _, e = w.Write(raw); e != nil { + return nil, e + } + if e = w.Close(); e != nil { + return nil, e + } + return b.Bytes(), nil +} +func decodeQueryCountRows(raw []byte) ([]domain.HumanItemRecord, error) { + r, e := gzip.NewReader(bytes.NewReader(raw)) + if e != nil { + return nil, e + } + defer r.Close() + b, e := io.ReadAll(io.LimitReader(r, 128<<20+1)) + if e != nil || len(b) > 128<<20 { + return nil, errors.New("invalid count cache") + } + var d struct { + Rows []domain.HumanItemRecord `bson:"rows"` + } + e = bson.Unmarshal(b, &d) + return d.Rows, e +} diff --git a/internal/store/query_replica_count_cache_test.go b/internal/store/query_replica_count_cache_test.go new file mode 100644 index 00000000..13d83ee4 --- /dev/null +++ b/internal/store/query_replica_count_cache_test.go @@ -0,0 +1,54 @@ +package store + +import ( + "droboticflow/internal/domain" + "errors" + "testing" +) + +func TestQueryReplicaCandidateCachePrivateMetadataAndIsolation(t *testing.T) { + original := domain.HumanItemRecord{ID: "i", LegacyAssignment: map[string]string{"role": "u"}, AccountHandoff: &domain.AccountTaskHandoff{Pending: true}, AcceptanceClaim: &domain.AcceptanceClaimAccess{UserID: "u", SupplierID: "s"}, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": "never copy"}}} + safe := cloneQueryCountRow(original) + raw, e := encodeQueryCountRows([]domain.HumanItemRecord{safe}) + if e != nil { + t.Fatal(e) + } + rows, e := decodeQueryCountRows(raw) + if e != nil || len(rows) != 1 { + t.Fatal(e) + } + if rows[0].Subject.Payload != nil || !rows[0].AccountHandoff.Pending || rows[0].AcceptanceClaim.UserID != "u" { + t.Fatal("private eligibility metadata lost") + } + private := cloneQueryCountRow(rows[0]) + private.AcceptanceClaim.SupplierID = "other" + private.AccountHandoff.Pending = false + private.LegacyAssignment["role"] = "other" + if rows[0].AcceptanceClaim.SupplierID != "s" || !rows[0].AccountHandoff.Pending || rows[0].LegacyAssignment["role"] != "u" { + t.Fatal("caller modified cached data") + } + if _, e = decodeQueryCountRows([]byte("invalid-cache")); e == nil { + t.Fatal("corrupt shared cache accepted") + } +} + +func TestQueryReplicaMissingConfigurationCannotPanicCoreCache(t *testing.T) { + t.Setenv("QUERY_REPLICA_REPORTS", "1") + t.Setenv("QUERY_REPLICA_POOLS", "1") + t.Setenv("QUERY_SOURCE_TRACKING", "0") + s := &MongoStore{} + close, e := s.ConfigureQueryReplicaFromEnvironment(t.Context()) + defer close() + if e == nil { + t.Fatal("serving without write tracking accepted") + } + if _, e = s.poolReplica(WithQueryReplicaPoolRead(t.Context())); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("invalid query configuration fell back to source pool", e) + } + if s.SharedQueryCache() != nil { + t.Fatal("invalid replica returned source cache") + } + if _, e = s.reportReplica(t.Context()); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal(e) + } +} diff --git a/internal/store/query_replica_delete.go b/internal/store/query_replica_delete.go new file mode 100644 index 00000000..ff6a7519 --- /dev/null +++ b/internal/store/query_replica_delete.go @@ -0,0 +1,121 @@ +package store + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// ReconcileDeleteStep handles one bounded part of a durable deletion intent. +// It reads only source identities, never annotator payloads. Survivors retain +// their data; a cancelled deletion is not replayed or restored from snapshots. +func (q *QueryReplica) ReconcileDeleteStep(ctx context.Context) (bool, error) { + var i queryDeleteIntent + e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"$or": bson.A{bson.M{"phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"phase": bson.M{"$in": bson.A{"prepared", "deleting"}}, "leaseUntil": bson.M{"$lte": time.Now().UTC()}}}}, options.FindOne().SetSort(bson.D{{Key: "audit", Value: 1}, {Key: "createdAt", Value: 1}})).Decode(&i) + if e == mongo.ErrNoDocuments { + return false, nil + } + if e != nil { + return false, e + } + if queryReplicaFields[i.Collection] == "" { + return false, ErrQueryReplicaUnsupported + } + if ok, e := q.Source.queryCollection(i.Collection).recoverDelete(ctx, i, time.Now().UTC()); e != nil || !ok { + return false, e + } + key := "delete/" + i.ID + var cp queryReplicaCheckpoint + e = q.control().FindOne(ctx, bson.M{"_id": key}).Decode(&cp) + if e != nil && e != mongo.ErrNoDocuments { + return false, e + } + if cp.Complete { + _, e = q.Source.db.Collection(querySyncDeleteCollection).UpdateOne(ctx, bson.M{"_id": i.ID, "phase": "reconcile"}, bson.M{"$set": bson.M{"phase": "acknowledged", "expireAt": time.Now().UTC().Add(7 * 24 * time.Hour)}}) + return true, e + } + filter := bson.M{} + if cp.HasLast || cp.Last != nil { + filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": cp.Last}}} + } + cur, e := q.Target.db.Collection(i.Collection).Find(ctx, filter, options.Find().SetProjection(bson.M{"_id": 1, "_queryReplica.sourceId": 1}).SetSort(bson.D{{Key: "_id", Value: 1}}).SetHint("_id_").SetLimit(q.limit())) + if e != nil { + return false, e + } + defer cur.Close(ctx) + var rows []bson.M + var ids bson.A + var last any + for cur.Next(ctx) { + var d bson.M + if e = cur.Decode(&d); e != nil { + return false, e + } + rows = append(rows, d) + last = d["_id"] + ids = append(ids, deletionDoc(d["_queryReplica"])["sourceId"]) + } + if e = cur.Err(); e != nil { + return false, e + } + present := []any{} + if len(ids) > 0 { + src, e := q.Source.db.Collection(i.Collection).Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_id": 1})) + if e != nil { + return false, e + } + for src.Next(ctx) { + var d bson.M + if e = src.Decode(&d); e != nil { + src.Close(ctx) + return false, e + } + present = append(present, d["_id"]) + } + e = src.Err() + src.Close(ctx) + if e != nil { + return false, e + } + } + absent := []bson.M{} + for _, r := range rows { + id := deletionDoc(r["_queryReplica"])["sourceId"] + exists := false + for _, v := range present { + if bsonEqual(id, v) { + exists = true + break + } + } + if !exists { + absent = append(absent, r) + } + } + e = q.transaction(ctx, func(sc mongo.SessionContext) error { + for _, r := range absent { + if _, e := q.Target.db.Collection(i.Collection).DeleteOne(sc, bson.M{"_id": r["_id"], "_queryReplica.sourceId": deletionDoc(r["_queryReplica"])["sourceId"]}); e != nil { + return e + } + } + if len(absent) > 0 { + if _, e := q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$inc": bson.M{"factEpoch": int64(1), "collectionEpoch." + i.Collection: int64(1)}}, options.Update().SetUpsert(true)); e != nil { + return e + } + } + d := bson.M{"complete": int64(len(rows)) < q.limit()} + if int64(len(rows)) < q.limit() { + d["expireAt"] = time.Now().UTC().Add(7 * 24 * time.Hour) + } + if len(rows) > 0 { + d["hasLast"] = true + d["last"] = last + } + _, e := q.control().UpdateOne(sc, bson.M{"_id": key}, bson.M{"$set": d}, options.Update().SetUpsert(true)) + return e + }) + return true, e +} diff --git a/internal/store/query_replica_indexes.go b/internal/store/query_replica_indexes.go new file mode 100644 index 00000000..2d20979d --- /dev/null +++ b/internal/store/query_replica_indexes.go @@ -0,0 +1,41 @@ +package store + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// EnsureTargetIndexes operates only on the explicitly supplied independent +// read database. No business migrations/bootstrapping run on this database. +func (q *QueryReplica) EnsureTargetIndexes(ctx context.Context) error { + for _, name := range replicaCollections() { + models := []mongo.IndexModel{{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetName("query_id_v1")}, {Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}, Options: options.Index().SetName("query_scope_v1")}} + if name == "annotationReviews" || name == "annotationRevisions" || name == "sliceRevisions" || name == "annotationWorklogEvents" { + models = append(models, mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "revision", Value: 1}}, Options: options.Index().SetName("query_task_v1")}) + } + if name == "humanItems" { + models = append(models, mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "pluginId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "status", Value: 1}, {Key: "projectId", Value: 1}, {Key: "assignedUserId", Value: 1}, {Key: "id", Value: 1}}, Options: options.Index().SetName("query_pool_stage_v1")}) + } + if name == "humanEvents" { + models = append(models, mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "actorUserId", Value: 1}, {Key: "eventType", Value: 1}, {Key: "itemId", Value: 1}}, Options: options.Index().SetName("query_reviewed_v1")}, mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}, {Key: "createdAt", Value: -1}}, Options: options.Index().SetName("query_item_event_v1")}) + } + + if _, e := q.Target.db.Collection(name).Indexes().CreateMany(ctx, models); e != nil { + return e + } + } + for _, group := range []map[string][]mongo.IndexModel{PoolSearchReadIndexes(), PlatformReadIndexes()} { + for name, models := range group { + if queryReplicaFields[name] == "" { + continue + } + if _, e := q.Target.db.Collection(name).Indexes().CreateMany(ctx, models); e != nil { + return e + } + } + } + _, e := q.control().Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "expireAt", Value: 1}}, Options: options.Index().SetName("query_replica_checkpoint_retention_v1").SetExpireAfterSeconds(0)}) + return e +} diff --git a/internal/store/query_replica_local_test.go b/internal/store/query_replica_local_test.go new file mode 100644 index 00000000..ccedc695 --- /dev/null +++ b/internal/store/query_replica_local_test.go @@ -0,0 +1,699 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "droboticflow/internal/workforcestats" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/sync/singleflight" +) + +func newLocalReplica(t *testing.T, monitors ...*event.CommandMonitor) (*MongoStore, *MongoStore, *QueryReplica) { + t.Helper() + source := newQuerySyncSource(t, monitors...) + c, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs")) + if e != nil { + t.Fatal(e) + } + db := c.Database(domain.NewID("query_replica_test")) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = db.Drop(ctx) + _ = c.Disconnect(ctx) + }) + target := NewMongoStore(db) + q, e := NewQueryReplica(source, target) + if e != nil { + t.Fatal(e) + } + if e = q.VerifyTopology(t.Context()); e != nil { + t.Fatal(e) + } + if e = q.EnsureSourceIndexes(t.Context()); e != nil { + t.Fatal(e) + } + if e = q.EnsureTargetIndexes(t.Context()); e != nil { + t.Fatal(e) + } + if e = q.Acquire(t.Context()); e != nil { + t.Fatal(e) + } + return source, target, q +} +func finishLocalReplica(t *testing.T, q *QueryReplica) { + t.Helper() + for n := 0; n < 100; n++ { + ready, e := q.RunOnce(t.Context(), true) + if e != nil { + t.Fatal(e) + } + if ready { + return + } + } + t.Fatal("projection did not drain") +} +func TestQueryReplicaBootstrapRestartMixedIDsAndPrivacyLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + q.BatchSize = 1 + for _, id := range []any{nil, "string-id", primitive.NewObjectID(), int32(7)} { + _, e := s.db.Collection("humanItems").InsertOne(ctx, bson.M{"_id": id, "id": domain.NewID("legacy"), "organizationId": "o", "projectId": "p", "subject": bson.M{"itemKey": "capture/part0001", "payload": bson.M{"annotationDraft": strings.Repeat("private annotation", 1000), "captureDirectory": "capture/part0001"}}}) + if e != nil { + t.Fatal(e) + } + } + if _, _, e := q.BootstrapStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + // Restart with the persisted checkpoint, including heterogeneous BSON ids. + q2, e := NewQueryReplica(s, d) + if e != nil { + t.Fatal(e) + } + q2.owner = q.owner + q2.epoch = q.epoch + q2.BatchSize = 1 + finishLocalReplica(t, q2) + n, e := d.humanItems.CountDocuments(ctx, bson.M{}) + if e != nil || n != 4 { + t.Fatal("bootstrap skipped legacy BSON id", n, e) + } + cur, e := d.humanItems.Find(ctx, bson.M{}) + if e != nil { + t.Fatal(e) + } + defer cur.Close(ctx) + for cur.Next(ctx) { + var doc bson.M + if e = cur.Decode(&doc); e != nil { + t.Fatal(e) + } + b, _ := json.Marshal(doc) + if strings.Contains(string(b), "private annotation") || strings.Contains(string(b), "annotationDraft") { + t.Fatal("customer annotation copied") + } + } + if e = cur.Err(); e != nil { + t.Fatal(e) + } + // Separate database names on the same server are not a CPU isolation boundary. + same, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true")) + if e != nil { + t.Fatal(e) + } + defer same.Disconnect(ctx) + bad, e := NewQueryReplica(s, NewMongoStore(same.Database("other_name"))) + if e != nil { + t.Fatal(e) + } + if bad.VerifyTopology(ctx) == nil { + t.Fatal("same process accepted") + } +} +func TestQueryReplicaConcurrentAcknowledgementAndLeaseFencingLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + c := s.queryCollection("humanItems") + if _, e := c.InsertOne(ctx, bson.M{"id": "i", "status": "ready", "version": 1}); e != nil { + t.Fatal(e) + } + rows, _, e := q.read(ctx, "humanItems", bson.M{querySyncField + ".dirty": true}) + if e != nil { + t.Fatal(e) + } + if e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, "humanItems", rows) }); e != nil { + t.Fatal(e) + } + // A save between sink commit and ack must remain pending. + if _, e = c.UpdateOne(ctx, bson.M{"id": "i", "version": 1}, bson.M{"$set": bson.M{"status": "submitted", "version": 2}}); e != nil { + t.Fatal(e) + } + if e = q.acknowledge(ctx, "humanItems", rows); e != nil { + t.Fatal(e) + } + var original bson.M + if e = c.FindOne(ctx, bson.M{"id": "i"}).Decode(&original); e != nil { + t.Fatal(e) + } + if deletionDoc(original[querySyncField])["dirty"] != true { + t.Fatal("concurrent save acknowledged") + } + if _, e = q.DrainStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + var replica bson.M + if e = d.humanItems.FindOne(ctx, bson.M{"id": "i"}).Decode(&replica); e != nil || replica["status"] != "submitted" { + t.Fatal(replica, e) + } + if e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, "humanItems", rows) }); e != nil { + t.Fatal(e) + } + if e = d.humanItems.FindOne(ctx, bson.M{"id": "i"}).Decode(&replica); e != nil || replica["status"] != "submitted" { + t.Fatal("old event overwrote current fact", e) + } + if _, e = q.control().UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}); e != nil { + t.Fatal(e) + } + next, e := NewQueryReplica(s, d) + if e != nil { + t.Fatal(e) + } + if e = next.Acquire(ctx); e != nil { + t.Fatal(e) + } + if e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, "humanItems", rows) }); !errors.Is(e, ErrQueryReplicaLease) { + t.Fatal("old process not fenced", e) + } +} +func TestQueryReplicaDeletionAndSourceAnnotationPreservationLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + c := s.queryCollection("humanItems") + for _, id := range []string{"delete", "keep"} { + if _, e := c.InsertOne(ctx, bson.M{"id": id, "version": 1, "subject": bson.M{"payload": bson.M{"annotationDraft": "keep latest"}}}); e != nil { + t.Fatal(e) + } + } + finishLocalReplica(t, q) + if _, e := c.DeleteOne(ctx, bson.M{"id": "delete", "version": 1}); e != nil { + t.Fatal(e) + } + finishLocalReplica(t, q) + if n, e := d.humanItems.CountDocuments(ctx, bson.M{"id": "delete"}); e != nil || n != 0 { + t.Fatal("deleted row retained", n, e) + } + var kept bson.M + if e := c.FindOne(ctx, bson.M{"id": "keep"}).Decode(&kept); e != nil { + t.Fatal(e) + } + if deletionDoc(deletionDoc(kept["subject"])["payload"])["annotationDraft"] != "keep latest" { + t.Fatal("surviving annotation changed") + } + // Competing delete claims must never overwrite the first intent's fencing nonce. + first, _, finish, e := c.prepareDelete(ctx, bson.M{"id": "keep"}, false, nil) + defer finish() + if e != nil { + t.Fatal(e) + } + _, _, stop, e := c.prepareDelete(ctx, bson.M{"id": "keep"}, false, nil) + stop() + if !errors.Is(e, errQueryDeleteFenced) { + t.Fatal("competing deletion did not fence", e) + } + if e = c.FindOne(ctx, bson.M{"id": "keep"}).Decode(&kept); e != nil { + t.Fatal(e) + } + if deletionDoc(kept[querySyncField])["deleteClaim"] != first.ID { + t.Fatal("first delete claim overwritten") + } +} +func TestQueryReplicaReportAndWorkRecordParityLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + seedAnnotationOutput(t, s) + ctx := t.Context() + if e := s.UpsertRun(ctx, domain.RunRecord{ID: "r", OrganizationID: "out", ProjectID: "p", TaskID: "c", CreatedAt: "2026-10-01T00:00:00Z", Status: domain.RunStatusFailed, Error: "采集 meta 文件身份冲突"}); e != nil { + t.Fatal(e) + } + if _, e := s.humanItems.UpdateOne(ctx, bson.M{"id": "i"}, bson.M{"$set": bson.M{"subject.payload.durationSeconds": 123.7}}); e != nil { + t.Fatal(e) + } + expectedDash, e := s.DashboardHumanWorkStats(ctx, "out") + if e != nil { + t.Fatal(e) + } + expectedOutput := outputStoreReport(t, s) + before, e := s.LoadProductionStats(ctx, "out", []string{"p"}) + if e != nil { + t.Fatal(e) + } + expectedProd := productionstats.Build(before) + if len(expectedProd.Sources) != 1 || expectedProd.Sources[0].ExceptionCategory != "metadata_identity_conflict" { + t.Fatal("exception fixture not exercised", expectedProd.Sources) + } + finishLocalReplica(t, q) + if e = s.ConfigureQueryReplicaReports(d, 30*time.Second); e != nil { + t.Fatal(e) + } + actualDash, e := s.DashboardHumanWorkStats(ctx, "out") + if e != nil || !reflect.DeepEqual(expectedDash, actualDash) { + t.Fatal("dashboard count/duration parity", expectedDash, actualDash, e) + } + actualOutput := outputStoreReport(t, s) + if !reflect.DeepEqual(expectedOutput, actualOutput) { + t.Fatalf("output facts differ: before=%+v after=%+v", expectedOutput.Totals, actualOutput.Totals) + } + after, e := s.LoadProductionStats(ctx, "out", []string{"p"}) + if e != nil { + t.Fatal(e) + } + actualProd := productionstats.Build(after) + if !reflect.DeepEqual(expectedProd, actualProd) { + t.Fatalf("production facts differ: before=%+v after=%+v", expectedProd, actualProd) + } + // Expired/outage state must not silently fall back to history scans on source. + _, e = d.db.Collection(queryReplicaControl).UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"observedAt": time.Time{}}}) + if e != nil { + t.Fatal(e) + } + if _, e = s.LoadProductionStats(ctx, "out", []string{"p"}); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("stale query replica fell back", e) + } + // Existing task detail remains on authority, regardless of report availability. + item, e := s.GetHumanItem(ctx, "i") + if e != nil || item.Subject.Payload["annotationDraft"] != "never export" { + t.Fatal("report routing changed task payload", item, e) + } +} + +func TestQueryReplicaWorkforceDetailsParityLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + seedWorkforceStats(t, s) + query := workforcestats.Query{SupplierID: "v", UserID: "u", Details: true, ProjectID: "p"} + query.From, _ = time.Parse(time.RFC3339, "2026-09-28T00:00:00Z") + query.To = query.From.Add(24 * time.Hour) + facts, e := s.LoadWorkforceStats(ctx, "workforce-test", query) + if e != nil { + t.Fatal(e) + } + expectedWork := workforcestats.Build(facts, query) + + finishLocalReplica(t, q) + if e = s.ConfigureQueryReplicaReports(d, 30*time.Second); e != nil { + t.Fatal(e) + } + facts, e = s.LoadWorkforceStats(ctx, "workforce-test", query) + if e != nil { + t.Fatal(e) + } + actualWork := workforcestats.Build(facts, query) + if !reflect.DeepEqual(expectedWork, actualWork) { + t.Fatalf("work record parity failed: before=%+v after=%+v", expectedWork, actualWork) + } + +} + +func TestQueryReplicaPoolSearchCountsAndCurrentAuthorityLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + for _, collection := range []string{"humanItems", "humanBatches", "annotationTasks", "runs", "projects"} { + docs := []any{} + switch collection { + case "projects": + docs = append(docs, bson.M{"id": "p", "organizationId": "o", "name": "项目"}) + case "runs": + docs = append(docs, bson.M{"id": "r", "organizationId": "o", "projectId": "p", "taskId": "c", "captureDirectories": bson.A{"FYLX_B07/part0002"}}) + case "humanBatches": + docs = append(docs, bson.M{"id": "b", "organizationId": "o", "projectId": "p", "stages": bson.A{bson.M{"id": "quality_review"}}}) + case "annotationTasks": + docs = append(docs, bson.M{"id": "t", "organizationId": "o", "projectId": "p", "runId": "r", "artifactId": "FYLX_B07_part0002.mp4", "status": "pending_quality"}) + case "humanItems": + for _, id := range []string{"i1", "i2", "i3"} { + docs = append(docs, bson.M{"id": id, "organizationId": "o", "projectId": "p", "runId": "r", "batchId": "b", "legacyTaskId": "t", "pluginId": "annotation-qa", "currentStageId": "quality_review", "status": "ready", "version": 1, "createdAt": "2026-11-01T00:00:00Z", "subject": bson.M{"itemKey": "FYLX_B07/part0002", "payload": bson.M{"annotationDraft": "existing annotation"}}}) + } + } + if _, e := s.queryCollection(collection).InsertMany(ctx, docs); e != nil { + t.Fatal(e) + } + } + tests := []domain.HumanItemQuery{{PluginID: "annotation-qa", PoolCandidateProjection: true, Limit: 2}, {PluginID: "annotation-qa", PoolCandidateProjection: true, Search: "B07", Limit: 2}, {PluginID: "annotation-qa", PoolCandidateProjection: true, Search: "part0002", Limit: 2}, {PluginID: "annotation-qa", PoolCandidateProjection: true, Search: "not-present", Limit: 2}} + expected := make([][]domain.HumanItemRecord, len(tests)) + counts := make([]int, len(tests)) + for i, test := range tests { + var e error + expected[i], e = s.ListHumanItems(ctx, "o", test) + if e != nil { + t.Fatal(e) + } + if e = s.VisitHumanPoolCountCandidates(ctx, "o", test, func(rows []domain.HumanItemRecord) error { counts[i] += len(rows); return nil }); e != nil { + t.Fatal(e) + } + } + finishLocalReplica(t, q) + if e := s.ConfigureQueryReplicaReports(d, 30*time.Second); e != nil { + t.Fatal(e) + } + s.EnableQueryReplicaPoolReads(true) + readCtx := WithQueryReplicaPoolRead(ctx) + for i, test := range tests { + actual, e := s.ListHumanItems(readCtx, "o", test) + if e != nil { + t.Fatal(e) + } + if !reflect.DeepEqual(expected[i], actual) { + t.Fatalf("pool list parity failed for query %d: expected=%+v actual=%+v", i, expected[i], actual) + } + n := 0 + if e = s.VisitHumanPoolCountCandidates(readCtx, "o", test, func(rows []domain.HumanItemRecord) error { n += len(rows); return nil }); e != nil || n != counts[i] { + t.Fatal("pool count mismatch", i, n, counts[i], e) + } + } + // Projection has not consumed this change yet. No stale assignment or version + // may be returned to the API's real-time access/canClaim calculation. + if _, e := s.humanItems.UpdateOne(ctx, bson.M{"id": "i3", "version": 1}, bson.M{"$set": bson.M{"assignedUserId": "other-user", "version": 2, "status": "claimed"}}); e != nil { + t.Fatal(e) + } + rows, e := s.ListHumanItems(readCtx, "o", tests[0]) + if e != nil { + t.Fatal(e) + } + if len(rows) != 2 || rows[0].AssignedUserID != "other-user" || rows[0].Version != 2 { + t.Fatal("stale claim/ownership reached authorization", rows) + } + // Normal active writes must not make filtered/blind counts unavailable. + for _, tc := range []struct { + search string + status domain.HumanItemStatus + actor string + ids []string + want int + }{ + {search: "B07", status: domain.HumanItemReady, want: 2}, + {search: "B07", status: domain.HumanItemClaimed, want: 1}, + {search: "not-present", want: 0}, + {actor: "other-user", want: 1}, + {actor: "unrelated-user", want: 0}, + {search: "B07", actor: "other-user", want: 1}, + {search: "B07", ids: []string{"i1"}, want: 1}, + } { + n := 0 + query := domain.HumanItemQuery{PluginID: "annotation-qa", Search: tc.search, Status: tc.status, BlindAnnotationActorID: tc.actor, IDs: tc.ids} + err := s.VisitHumanPoolCountCandidates(WithQueryReplicaCountRead(ctx), "o", query, func(rows []domain.HumanItemRecord) error { n += len(rows); return nil }) + if err != nil || n != tc.want { + t.Fatalf("pending row count %+v: got %d, err %v", tc, n, err) + } + } + if _, e = s.humanItems.DeleteOne(ctx, bson.M{"id": "i2"}); e != nil { + t.Fatal(e) + } + rows, e = s.ListHumanItems(readCtx, "o", tests[0]) + if e != nil { + t.Fatal(e) + } + if len(rows) != 2 || !rows[1].QueryReplicaOmit || rows[1].ID != "i2" { + t.Fatal("deletion truncated pagination cursor", rows) + } + if e = s.queryCollection("humanItems").Drop(ctx); !errors.Is(e, ErrQueryReplicaUnsupported) { + t.Fatal("unobservable DDL allowed", e) + } +} + +func TestQueryReplicaOldWriterReconciliationAndRevisionResetLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + c := s.queryCollection("humanItems") + if _, e := c.InsertOne(ctx, bson.M{"id": "i", "version": 1, "status": "ready", "subject": bson.M{"itemKey": "capture/part0001", "payload": bson.M{"annotationDraft": "first draft"}}}); e != nil { + t.Fatal(e) + } + finishLocalReplica(t, q) + // Simulate an in-flight old writer: replacement does not know private markers. + if _, e := c.Collection.ReplaceOne(ctx, bson.M{"id": "i", "version": 1}, bson.M{"id": "i", "version": 2, "status": "submitted", "subject": bson.M{"itemKey": "capture/part0001", "payload": bson.M{"annotationDraft": "latest rework"}}}); e != nil { + t.Fatal(e) + } + for pass := 0; pass < 50; pass++ { + if _, e := q.ReconcileOneDue(ctx, 0); e != nil { + t.Fatal(e) + } + if _, e := q.DrainStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + } + var source, replica bson.M + if e := c.FindOne(ctx, bson.M{"id": "i"}).Decode(&source); e != nil { + t.Fatal(e) + } + if e := d.humanItems.FindOne(ctx, bson.M{"id": "i"}).Decode(&replica); e != nil { + t.Fatal(e) + } + if replica["status"] != "submitted" || deletionDoc(deletionDoc(source["subject"])["payload"])["annotationDraft"] != "latest rework" || supplierNumber(source["version"]) != 2 { + t.Fatal("old writer recovery lost facts or changed annotation/version") + } + // A dirty old revision must not be acknowledged over a newer durable target. + if _, e := c.Collection.UpdateOne(ctx, bson.M{"id": "i"}, bson.M{"$set": bson.M{querySyncField + ".revision": int64(1), querySyncField + ".dirty": true, "status": "claimed"}}); e != nil { + t.Fatal(e) + } + if _, e := q.DrainStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + if e := c.FindOne(ctx, bson.M{"id": "i"}).Decode(&source); e != nil { + t.Fatal(e) + } + if deletionDoc(source[querySyncField])["dirty"] != true || querySyncRev(deletionDoc(source[querySyncField])) <= 1 { + t.Fatal("revision reset silently acknowledged") + } + if _, e := q.DrainStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + if e := d.humanItems.FindOne(ctx, bson.M{"id": "i"}).Decode(&replica); e != nil || replica["status"] != "claimed" { + t.Fatal("revision reset not repaired", e) + } + if _, e := c.Collection.DeleteOne(ctx, bson.M{"id": "i"}); e != nil { + t.Fatal(e) + } + for pass := 0; pass < 80; pass++ { + if _, e := q.ReconcileOneDue(ctx, 0); e != nil { + t.Fatal(e) + } + if _, e := q.ReconcileDeleteStep(ctx); e != nil { + t.Fatal(e) + } + } + if n, e := d.humanItems.CountDocuments(ctx, bson.M{"id": "i"}); e != nil || n != 0 { + t.Fatal("unreported deletion retained forever", n, e) + } +} +func TestQueryReplicaRejectsSignedLegacyNamesWithoutLeaking(t *testing.T) { + d := bson.M{"_id": "physical", "id": "i", "subject": bson.M{"payload": bson.M{"captureDirectory": "https://user:private@host/capture?secret=value"}}} + r, e := queryExportRecord("humanItems", d) + if !errors.Is(e, ErrQueryReplicaUnsupported) || r.Document != nil { + t.Fatal("unsafe legacy name copied", e) + } +} + +func TestQueryReplicaCountCacheInvalidationAndCallerCancellationLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + ctx := t.Context() + docs := []any{} + for n := 0; n < 5000; n++ { + docs = append(docs, bson.M{"id": fmt.Sprint("i", n), "organizationId": "o", "pluginId": "annotation-qa", "status": "ready"}) + } + if _, e := s.humanItems.InsertMany(ctx, docs); e != nil { + t.Fatal(e) + } + for pass := 0; pass < 100; pass++ { + ready, e := q.RunOnce(ctx, true) + if e != nil { + t.Fatal(e) + } + if ready { + break + } + } + if e := s.ConfigureQueryReplicaReports(d, time.Minute); e != nil { + t.Fatal(e) + } + s.EnableQueryReplicaPoolReads(true) + query := domain.HumanItemQuery{PluginID: "annotation-qa", Status: domain.HumanItemReady} + count := func(ctx context.Context) (int, error) { + n := 0 + e := s.VisitHumanPoolCountCandidates(WithQueryReplicaCountRead(ctx), "o", query, func(rows []domain.HumanItemRecord) error { + n += len(rows) + if len(rows) > 0 { + rows[0].AssignedUserID = "must not leak" + } + return nil + }) + return n, e + } + short, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { _, e := count(short); done <- e }() + cancel() + n, e := count(ctx) + if e != nil || n != 5000 { + t.Fatal("cancelled caller stopped independent reader", n, e) + } + if e = <-done; !errors.Is(e, context.Canceled) { + t.Fatal(e) + } + // Before projection catches up, the pending source row is overlaid/excluded. + if _, e = s.humanItems.UpdateOne(ctx, bson.M{"id": "i0"}, bson.M{"$set": bson.M{"status": "claimed"}}); e != nil { + t.Fatal(e) + } + n, e = count(ctx) + if e != nil || n != 4999 { + t.Fatal("warm cache ignored authoritative pending change", n, e) + } + if _, e = q.DrainStep(ctx, "humanItems"); e != nil { + t.Fatal(e) + } + n, e = count(ctx) + if e != nil || n != 4999 { + t.Fatal("fact epoch did not invalidate warm cache", n, e) + } + if _, e = d.db.Collection(queryReplicaControl).UpdateOne(ctx, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}); e != nil { + t.Fatal(e) + } + if _, e = count(ctx); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("cache bypassed query-store readiness", e) + } +} + +func TestQueryReplicaEnvironmentUsesIndependentFactsLocal(t *testing.T) { + t.Setenv("QUERY_SERVING_NAMESPACE", domain.NewID("query-env")) + s, d, q := newLocalReplica(t) + finishLocalReplica(t, q) + t.Setenv("QUERY_SOURCE_TRACKING", "1") + t.Setenv("QUERY_REPLICA_REPORTS", "1") + t.Setenv("QUERY_REPLICA_POOLS", "1") + t.Setenv("QUERY_READ_MONGO_URI", "mongodb://127.0.0.1:27129/?replicaSet=query-read-rs") + t.Setenv("QUERY_READ_MONGO_DB", d.db.Name()) + t.Setenv("QUERY_SERVING_READS", "0") + close, e := s.ConfigureQueryReplicaFromEnvironment(t.Context()) + defer close() + if e != nil { + t.Fatal(e) + } + target, e := s.poolReplica(WithQueryReplicaPoolRead(t.Context())) + if e != nil || target == nil || !target.queryReplicaFactStore || target.db.Name() != d.db.Name() { + t.Fatal("environment routing did not select independent thin fact reads", e) + } +} + +func TestQueryReplicaBindingRejectsDifferentAuthorityAndNamespaceLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + finishLocalReplica(t, q) + other := NewMongoStore(s.db.Client().Database(s.db.Name() + "-other")) + if e := other.ConfigureQueryReplicaReports(d, time.Minute); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("wrong authority accepted", e) + } + wrong, e := NewQueryReplica(other, d) + if e != nil { + t.Fatal(e) + } + if e = wrong.Acquire(t.Context()); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("projector rebound existing query data", e) + } + t.Setenv("QUERY_SERVING_NAMESPACE", "different-deployment") + if e = s.ConfigureQueryReplicaReports(d, time.Minute); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("wrong namespace accepted", e) + } +} + +type gatedQueryDialer struct { + blocked atomic.Bool + dialer net.Dialer +} + +func (d *gatedQueryDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if d.blocked.Load() { + return nil, errors.New("isolated query connection unavailable") + } + return d.dialer.DialContext(ctx, network, address) +} +func TestQueryReplicaColdConnectionRecoversWithoutAPIReconfigurationLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + finishLocalReplica(t, q) + dialer := &gatedQueryDialer{} + dialer.blocked.Store(true) + client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs").SetDialer(dialer).SetServerSelectionTimeout(100*time.Millisecond).SetConnectTimeout(100*time.Millisecond)) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { _ = client.Disconnect(context.Background()) }) + target := NewMongoStore(client.Database(d.db.Name())) + pending, e := newQueryReplicaReads(s, target, time.Minute) + if e != nil { + t.Fatal(e) + } + s.queryReplicaReads = pending + if _, e = s.reportReplica(t.Context()); !errors.Is(e, ErrQueryReplicaNotReady) { + t.Fatal("cold query did not fail closed", e) + } + if pending.verified.Load() { + t.Fatal("unreachable query verified") + } + dialer.blocked.Store(false) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + got, e := s.reportReplica(t.Context()) + if e == nil && got == target { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatal("query did not recover without replacing/reconfiguring API reader") +} + +func TestQueryReplicaWatermarkBurstUsesOneMongoCommandLocal(t *testing.T) { + s, d, q := newLocalReplica(t) + finishLocalReplica(t, q) + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + var reads atomic.Int64 + monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) { + if e.CommandName == "find" && e.Command.Lookup("find").StringValue() == queryReplicaControl { + var cmd struct { + Filter struct { + ID string `bson:"_id"` + } `bson:"filter"` + } + if bson.Unmarshal(e.Command, &cmd) == nil && cmd.Filter.ID == "state" { + reads.Add(1) + once.Do(func() { close(entered); <-release }) + } + } + }} + client, e := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27129/?replicaSet=query-read-rs").SetMonitor(monitor)) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { _ = client.Disconnect(context.Background()) }) + target := NewMongoStore(client.Database(d.db.Name())) + r, e := newQueryReplicaReads(s, target, time.Minute) + if e != nil { + t.Fatal(e) + } + channels := []<-chan singleflight.Result{r.stateResult()} + select { + case <-entered: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("metadata probe did not start") + } + for n := 1; n < 100; n++ { + channels = append(channels, r.stateResult()) + } + close(release) + for _, ch := range channels { + select { + case result := <-ch: + if result.Err != nil || !result.Val.(queryReplicaReadState).Ready { + t.Fatal("shared probe failed", result.Err) + } + case <-time.After(5 * time.Second): + t.Fatal("metadata follower stuck") + } + } + if reads.Load() != 1 { + t.Fatal("burst repeated identical Mongo readiness reads", reads.Load()) + } +} diff --git a/internal/store/query_replica_pool.go b/internal/store/query_replica_pool.go new file mode 100644 index 00000000..816726c9 --- /dev/null +++ b/internal/store/query_replica_pool.go @@ -0,0 +1,109 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type queryReplicaPoolReadKey struct{} + +// WithQueryReplicaPoolRead is used only by list/count GET handlers, never by +// claims, session renewals or business jobs looking for executable work. +func WithQueryReplicaPoolRead(ctx context.Context) context.Context { + if _, ok := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier); ok { + return ctx + } + return context.WithValue(ctx, queryReplicaPoolReadKey{}, &queryReplicaPoolBarrier{}) +} +func (s *MongoStore) UsesQueryReplicaPool(ctx context.Context) bool { + _, v := ctx.Value(queryReplicaPoolReadKey{}).(*queryReplicaPoolBarrier) + return v && s.queryReplicaReads != nil && s.queryReplicaReads.pool +} +func (s *MongoStore) poolReplica(ctx context.Context) (*MongoStore, error) { + if !s.UsesQueryReplicaPool(ctx) { + return nil, nil + } + return s.reportReplica(ctx) +} +func (s *MongoStore) EnableQueryReplicaPoolReads(enabled bool) { + if s.queryReplicaReads != nil { + s.queryReplicaReads.pool = enabled + } +} + +// Query candidates come from the read store; current task scope/status/version +// are fetched by bounded IDs before API authorization and canClaim computation. +// Missing/changed candidates keep an internal cursor placeholder so deletion +// cannot falsely end pagination. They never enter the public response. +func (s *MongoStore) replicaPoolCandidates(ctx context.Context, target *MongoStore, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) { + rows, e := target.ListHumanItems(ctx, org, q) + if e != nil || len(rows) == 0 { + return rows, e + } + ids := make([]string, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.ID) + } + current := map[string]domain.HumanItemRecord{} + for start := 0; start < len(ids); start += 200 { + end := start + 200 + if end > len(ids) { + end = len(ids) + } + cur, e := s.humanItems.Find(ctx, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": ids[start:end]}}), options.Find().SetProjection(humanItemListProjection(domain.HumanItemQuery{PoolCandidateProjection: true}))) + if e != nil { + return nil, e + } + for cur.Next(ctx) { + var item domain.HumanItemRecord + if e = cur.Decode(&item); e != nil { + cur.Close(ctx) + return nil, e + } + current[item.ID] = item + } + e = cur.Err() + cur.Close(ctx) + if e != nil { + return nil, e + } + } + for i, old := range rows { + item, exists := current[old.ID] + if !exists || !humanItemMatches(item, q) { + rows[i] = domain.HumanItemRecord{ID: old.ID, CreatedAt: old.CreatedAt, QueryReplicaOmit: true} + continue + } + rows[i] = item + } + return rows, nil +} + +func (s *MongoStore) replicaAdminPool(ctx context.Context, target *MongoStore, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) { + rows, e := target.ListAdminAnnotationPool(ctx, org, q) + if e != nil || len(rows) == 0 { + return rows, e + } + ids := make([]string, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.ID) + } + current, e := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids) + if e != nil { + return nil, e + } + if len(current) != len(rows) { + return nil, ErrQueryReplicaNotReady + } + if e = s.EnrichAnnotationSourceStatuses(ctx, current); e != nil { + return nil, e + } + for _, item := range current { + if !adminPoolScope(item) || !adminPoolSelected(item, q) { + return nil, ErrQueryReplicaNotReady + } + } + return current, nil +} diff --git a/internal/store/query_replica_projection.go b/internal/store/query_replica_projection.go new file mode 100644 index 00000000..d16b2203 --- /dev/null +++ b/internal/store/query_replica_projection.go @@ -0,0 +1,175 @@ +package store + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/url" + "strings" + + "droboticflow/internal/domain" + "droboticflow/internal/productionstats" + "go.mongodb.org/mongo-driver/bson" +) + +var ErrQueryReplicaUnsupported = errors.New("query replica record requires compatibility review") + +type queryReplicaRecord struct { + SourceID any + Revision int64 + TargetRevision int64 + TargetSourceID any + Document bson.M +} + +func querySourceProjection(collection string) bson.M { + p := queryReplicaProjection(collection) + if collection == "humanItems" { + p["__legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression(collection), bson.M{}, "$subject.payload"}} + } + return p +} +func safeReplicaNames(d any) bool { + switch value := d.(type) { + case string: + u, e := url.Parse(value) + if e != nil { + return !strings.Contains(value, "://") && !strings.HasPrefix(value, "//") + } + return u.Host == "" || (u.User == nil && u.RawQuery == "" && u.Fragment == "") + case bson.M: + for _, v := range value { + if !safeReplicaNames(v) { + return false + } + } + case map[string]any: + for _, v := range value { + if !safeReplicaNames(v) { + return false + } + } + case bson.A: + for _, v := range value { + if !safeReplicaNames(v) { + return false + } + } + case []any: + for _, v := range value { + if !safeReplicaNames(v) { + return false + } + } + } + return true +} +func queryExportRecord(collection string, d bson.M) (queryReplicaRecord, error) { + source := d["_id"] + meta := deletionDoc(d[querySyncField]) + revision := int64(supplierNumber(meta["revision"])) + delete(d, querySyncField) + if collection == "humanItems" { + job := deletionDoc(d["processingJob"]) + if raw, ok := job["error"].(string); ok { + if strings.HasPrefix(raw, AnnotationSourceUnavailablePrefix) { + job["error"] = AnnotationSourceUnavailablePrefix + } else { + delete(job, "error") + } + d["processingJob"] = job + } + legacy := deletionDoc(d["__legacyPayload"]) + delete(d, "__legacyPayload") + if len(legacy) > 0 { + subject := deletionDoc(d["subject"]) + thin := subject["payload"] + subject["payload"] = legacy + d["subject"] = subject + stampPoolSearchMetadata(collection, d) + subject["payload"] = thin + } + } + if collection == "runs" { + var run domain.RunRecord + b, e := bson.Marshal(d) + if e != nil { + return queryReplicaRecord{}, e + } + if e = bson.Unmarshal(b, &run); e != nil { + return queryReplicaRecord{}, e + } + d["queryExceptionCategory"] = productionstats.ExceptionCategory(run) + delete(d, "error") + if d["failure"] != nil { + d["failure"] = bson.M{} + } + // Derive legacy names before dropping all non-reference input values. + if d["poolSearch"] == nil { + stampPoolSearchMetadata(collection, d) + } + inputs := bson.M{} + for key, value := range deletionDoc(d["inputs"]) { + ref := deletionDoc(deletionDoc(value)["ref"]) + safe := bson.M{} + for _, field := range []string{"captureUnit", "prefix", "key", "manifestKey"} { + if v, ok := ref[field]; ok { + safe[field] = v + } + } + inputs[key] = bson.M{"ref": safe} + } + if d["inputs"] != nil { + d["inputs"] = inputs + } + } + encoded, err := bson.Marshal(d) + if err != nil { + return queryReplicaRecord{}, err + } + var normalized bson.M + if err = bson.Unmarshal(encoded, &normalized); err != nil { + return queryReplicaRecord{}, err + } + d = normalized + if !safeReplicaNames(d) { + return queryReplicaRecord{}, ErrQueryReplicaUnsupported + } + if logical, ok := d["id"].(string); ok && strings.TrimSpace(logical) != "" { + d["_id"] = logical + } + d["_queryReplica"] = bson.M{"sourceId": source, "revision": revision, "schema": queryReplicaSchema, "digest": queryReplicaDigest(collection, d)} + return queryReplicaRecord{SourceID: source, Revision: revision, Document: d}, nil +} + +// A semantic fact digest ignores edits that cannot change query results. In +// particular a draft save must not turn low-frequency reconciliation into a +// second stream of business writes. +func queryReplicaDigest(collection string, d bson.M) string { + copy := bson.M{} + for k, v := range d { + if k != "_id" && k != "_queryReplica" && k != "updatedAt" && k != "version" { + copy[k] = v + } + } + if collection == "humanItems" && d["pendingCombinedSubmissionId"] != nil && d["pendingCombinedSubmissionId"] != "" { + copy["version"] = d["version"] + } + if p := deletionDoc(d["poolSearch"]); len(p) > 0 { + m := bson.M{} + for _, k := range []string{"text", "payloadDirectories", "durationSeconds", "searchText", "textVersion", "schema"} { + if v, ok := p[k]; ok { + m[k] = v + } + } + copy["poolSearch"] = m + } + + b, e := json.Marshal(copy) + if e != nil { + return "" + } + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} diff --git a/internal/store/query_replica_reads.go b/internal/store/query_replica_reads.go new file mode 100644 index 00000000..740bd591 --- /dev/null +++ b/internal/store/query_replica_reads.go @@ -0,0 +1,281 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/workforcestats" + "errors" + "os" + "sync/atomic" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/sync/singleflight" +) + +type queryReplicaReads struct { + stateFlights singleflight.Group + verifier *QueryReplica + verified atomic.Bool + verifyPermit chan struct{} + nextVerify time.Time + pool bool + target *MongoStore + maxAge time.Duration +} + +func newQueryReplicaReads(source, target *MongoStore, maxAge time.Duration) (*queryReplicaReads, error) { + if source == nil || source.db == nil || target == nil || target == source || target.queryReplicaReads != nil || maxAge <= 0 || maxAge > time.Minute { + return nil, errors.New("invalid query replica routing") + } + verifier, e := NewQueryReplica(source, target) + if e != nil { + return nil, e + } + r := &queryReplicaReads{verifier: verifier, target: target, maxAge: maxAge, verifyPermit: make(chan struct{}, 1)} + r.verifyPermit <- struct{}{} + return r, nil +} + +// ConfigureQueryReplicaReports changes report readers only. Authentication, +// project/supplier authorization, task details and every mutation stay on s. +// Missing/stale replicas return unavailable; they never stampede the source. +// This is startup-only, after explicit shadow/result/coverage acceptance. +func (s *MongoStore) ConfigureQueryReplicaReports(target *MongoStore, maxAge time.Duration) error { + r, e := newQueryReplicaReads(s, target, maxAge) + if e != nil { + return e + } + if e = s.verifyReplicaBinding(target); e != nil { + return e + } + s.queryReplicaReads = r + return nil +} + +// A valid but cold query service retains its driver client. One cancelable +// verifier retries at most once per second; no source history fallback and no +// unsynchronized replacement of the active reader pointer is involved. +func (s *MongoStore) verifyReplicaRead(ctx context.Context, r *queryReplicaReads) error { + if r.verified.Load() { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.verifyPermit: + } + defer func() { r.verifyPermit <- struct{}{} }() + if r.verified.Load() { + return nil + } + if time.Now().Before(r.nextVerify) { + return ErrQueryReplicaNotReady + } + short, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + e := r.verifier.VerifyTopology(short) + if e == nil { + e = s.verifyReplicaBindingContext(short, r.target) + } + if e != nil { + r.nextVerify = time.Now().Add(time.Second) + if ctx.Err() != nil { + return ctx.Err() + } + return ErrQueryReplicaNotReady + } + r.verified.Store(true) + return nil +} + +func (s *MongoStore) reportReplica(ctx context.Context) (*MongoStore, error) { + if e := ctx.Err(); e != nil { + return nil, e + } + if s.queryReplicaReads == nil { + return nil, nil + } + if s.queryReplicaReads.target == nil { + return nil, ErrQueryReplicaNotReady + } + if e := s.verifyReplicaRead(ctx, s.queryReplicaReads); e != nil { + return nil, e + } + var state queryReplicaReadState + var e error + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-s.queryReplicaReads.stateResult(): + e = result.Err + if e == nil { + state = result.Val.(queryReplicaReadState) + } + } + if e := ctx.Err(); e != nil { + return nil, e + } + age := time.Since(state.ObservedAt) + if e != nil || !state.Ready || state.Schema != queryReplicaSchema || state.Binding.SourceDatabase != s.db.Name() || state.Binding.Namespace != queryReplicaNamespace(s.db.Name()) || age < 0 || age > s.queryReplicaReads.maxAge { + return nil, ErrQueryReplicaNotReady + } + return s.queryReplicaReads.target, nil +} + +type queryReplicaReadState struct { + Binding queryReplicaBinding `bson:",inline"` + Ready bool `bson:"ready"` + Schema string `bson:"schema"` + ObservedAt time.Time `bson:"observedAt"` +} + +// Coalesce only in-flight metadata reads. There is no cross-request TTL or +// cached permission decision; every completed probe ends this shared flight. +// One canceled browser cannot cancel other readers' bounded verification. +func (r *queryReplicaReads) stateResult() <-chan singleflight.Result { + return r.stateFlights.DoChan("watermark", func() (any, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + var state queryReplicaReadState + e := r.target.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}).Decode(&state) + return state, e + }) +} + +// PublishReady is called only after an indexed, complete drain of all source +// collections AND deletion intents, never merely on process liveness. +func (q *QueryReplica) PublishReady(ctx context.Context) error { + for _, n := range replicaCollections() { + cp, e := q.checkpoint(ctx, n) + if e != nil { + return e + } + if !cp.Complete { + return ErrQueryReplicaNotReady + } + e = q.Source.db.Collection(n).FindOne(ctx, bson.M{querySyncField + ".dirty": true}, options.FindOne().SetProjection(bson.M{"_id": 1}).SetHint("query_sync_dirty_v1")).Err() + if e == nil { + return ErrQueryReplicaNotReady + } + if e != mongo.ErrNoDocuments { + return e + } + } + e := q.Source.db.Collection(querySyncDeleteCollection).FindOne(ctx, bson.M{"audit": bson.M{"$ne": true}, "phase": bson.M{"$in": queryDeletePendingPhases()}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err() + if e == nil { + return ErrQueryReplicaNotReady + } + if e != mongo.ErrNoDocuments { + return e + } + return q.transaction(ctx, func(sc mongo.SessionContext) error { + _, e := q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": true, "schema": queryReplicaSchema, "observedAt": time.Now().UTC(), "sourceDatabase": q.binding().SourceDatabase, "namespace": q.binding().Namespace}}, options.Update().SetUpsert(true)) + return e + }) +} + +func (s *MongoStore) loadReplicaWorkforce(ctx context.Context, target *MongoStore, org string, q workforcestats.Query) (workforcestats.Snapshot, error) { + details := q.Details + d, e := target.LoadWorkforceStatsCached(ctx, org, q) + if e != nil || !details { + return d, e + } + // Only immutable, already-authorized result identities are hydrated. No + // source history scan or draft/media payload is used to serve this request. + for start := 0; start < len(d.Operations); start += 200 { + end := start + 200 + if end > len(d.Operations) { + end = len(d.Operations) + } + ids := []string{} + for _, v := range d.Operations[start:end] { + ids = append(ids, v.ID) + } + cur, e := s.db.Collection("reviewOperations").Find(ctx, organizationScopedFilter(org, bson.M{"_id": bson.M{"$in": ids}, "status": "completed"}), options.Find().SetProjection(bson.M{"_id": 1, "draft.problems.text": 1, "effects.review.comment": 1, "effects.reject.comment": 1})) + if e != nil { + return d, e + } + var texts []domain.ReviewOperation + e = cur.All(ctx, &texts) + cur.Close(ctx) + if e != nil { + return d, e + } + byID := map[string]domain.ReviewOperation{} + for _, v := range texts { + byID[v.ID] = v + } + for i := start; i < end; i++ { + v, ok := byID[d.Operations[i].ID] + if !ok { + continue + } + o := &d.Operations[i] + for j := range o.Draft.Problems { + if j < len(v.Draft.Problems) { + o.Draft.Problems[j].Text = v.Draft.Problems[j].Text + } + } + for j := range o.Effects { + if j >= len(v.Effects) { + continue + } + src := v.Effects[j] + dst := &o.Effects[j] + if src.Review != nil && dst.Review != nil { + dst.Review.Comment = src.Review.Comment + } + if src.Reject != nil && dst.Reject != nil { + dst.Reject.Comment = src.Reject.Comment + } + } + } + } + for start := 0; start < len(d.Reviews); start += 200 { + end := start + 200 + if end > len(d.Reviews) { + end = len(d.Reviews) + } + ids := []string{} + for _, v := range d.Reviews[start:end] { + ids = append(ids, v.ID) + } + cur, e := s.annotationReviews.Find(ctx, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": ids}}), options.Find().SetProjection(bson.M{"id": 1, "comment": 1})) + if e != nil { + return d, e + } + var rows []domain.AnnotationReviewRecord + e = cur.All(ctx, &rows) + cur.Close(ctx) + if e != nil { + return d, e + } + texts := map[string]string{} + for _, v := range rows { + texts[v.ID] = v.Comment + } + for i := start; i < end; i++ { + d.Reviews[i].Comment = texts[d.Reviews[i].ID] + } + } + return d, nil +} + +// VerifyQueryReplicaTarget prevents the aggregation worker from accidentally +// returning to an authority database. Only the projector creates this identity. +func (s *MongoStore) VerifyQueryReplicaTarget(ctx context.Context) error { + var state struct { + Binding queryReplicaBinding `bson:",inline"` + Schema string `bson:"schema"` + } + if e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}).Decode(&state); e != nil { + return e + } + if state.Schema != queryReplicaSchema || state.Binding.SourceDatabase == "" || state.Binding.Namespace != os.Getenv("QUERY_SERVING_NAMESPACE") || state.Binding.Namespace == "" { + return ErrQueryReplicaNotReady + } + return nil +} diff --git a/internal/store/query_replica_reconcile.go b/internal/store/query_replica_reconcile.go new file mode 100644 index 00000000..b398d430 --- /dev/null +++ b/internal/store/query_replica_reconcile.go @@ -0,0 +1,124 @@ +package store + +import ( + "context" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func queryReplicaSourceCAS(r queryReplicaRecord) bson.M { + f := bson.M{"_id": r.SourceID} + if r.Revision > 0 { + f[querySyncField+".revision"] = r.Revision + } else { + f[querySyncField+".revision"] = bson.M{"$exists": false} + } + return f +} +func queryReplicaRepairMarker(revision int64) bson.M { + return bson.M{querySyncField + ".revision": revision, querySyncField + ".dirty": true, querySyncField + ".signature": "", querySyncField + ".mode": "unknown", querySyncField + ".producer": "query-source/v1"} +} + +// ReconcileOneDue performs at most one bounded metadata batch. It catches old +// maintenance jobs that didn't tag changes, without a full per-page source +// scan. It repairs private marker fields only; annotations and business +// versions are neither copied back nor changed. Writer coverage is still a +// cutover prerequisite; reconciliation is recovery, not a CDC freshness claim. +func (q *QueryReplica) ReconcileOneDue(ctx context.Context, interval time.Duration) (int, error) { + names := replicaCollections() + for inspected := 0; inspected < len(names); inspected++ { + name := names[q.reconcileNext%len(names)] + q.reconcileNext = (q.reconcileNext + 1) % len(names) + boot, e := q.checkpoint(ctx, name) + if e != nil { + return 0, e + } + if !boot.Complete { + continue + } + key := "reconcile/" + name + var cp queryReplicaCheckpoint + e = q.control().FindOne(ctx, bson.M{"_id": key}).Decode(&cp) + if e != nil && e != mongo.ErrNoDocuments { + return 0, e + } + if !cp.HasLast && cp.Last == nil && !cp.CheckedAt.IsZero() && time.Since(cp.CheckedAt) < interval { + continue + } + filter := bson.M{} + if cp.HasLast || cp.Last != nil { + filter["$expr"] = bson.M{"$gt": bson.A{"$_id", bson.M{"$literal": cp.Last}}} + } + rows, last, e := q.read(ctx, name, filter) + if e != nil { + return 0, e + } + ids := []any{} + for _, r := range rows { + ids = append(ids, r.Document["_id"]) + } + target := map[string]bson.M{} + if len(ids) > 0 { + cur, e := q.Target.db.Collection(name).Find(ctx, bson.M{"_id": bson.M{"$in": ids}}, options.Find().SetProjection(bson.M{"_queryReplica": 1})) + if e != nil { + return 0, e + } + for cur.Next(ctx) { + var d bson.M + if e = cur.Decode(&d); e != nil { + cur.Close(ctx) + return 0, e + } + target[querySourceIdentity(name, d["_id"])] = deletionDoc(d["_queryReplica"]) + } + e = cur.Err() + cur.Close(ctx) + if e != nil { + return 0, e + } + } + models := []mongo.WriteModel{} + for _, r := range rows { + meta := target[querySourceIdentity(name, r.Document["_id"])] + if bsonEqual(meta["sourceId"], r.SourceID) && meta["schema"] == queryReplicaSchema && meta["digest"] == deletionDoc(r.Document["_queryReplica"])["digest"] { + continue + } + next := max(r.Revision, int64(supplierNumber(meta["revision"]))) + 1 + models = append(models, mongo.NewUpdateOneModel().SetFilter(queryReplicaSourceCAS(r)).SetUpdate(bson.M{"$set": queryReplicaRepairMarker(next)})) + } + if len(models) > 0 { + if e = q.transaction(ctx, func(sc mongo.SessionContext) error { + _, e := q.control().UpdateOne(sc, bson.M{"_id": "state"}, bson.M{"$set": bson.M{"ready": false}}, options.Update().SetUpsert(true)) + return e + }); e != nil { + return 0, e + } + if _, e = q.Source.db.Collection(name).BulkWrite(ctx, models, options.BulkWrite().SetOrdered(false)); e != nil { + return 0, e + } + } + done := int64(len(rows)) < q.limit() + if done { + // Reuse the durable identity sweeper for unreported physical deletions. + // Audits have lower priority and do not block unrelated report readiness. + i := queryDeleteIntent{ID: domain.NewID("query-audit"), Collection: name, Phase: "reconcile", CreatedAt: time.Now().UTC(), Audit: true} + if _, e = q.Source.db.Collection(querySyncDeleteCollection).InsertOne(ctx, i); e != nil { + return 0, e + } + } + e = q.transaction(ctx, func(sc mongo.SessionContext) error { + update := bson.M{"$set": bson.M{"last": last, "hasLast": true}} + if done { + update = bson.M{"$set": bson.M{"checkedAt": time.Now().UTC()}, "$unset": bson.M{"last": "", "hasLast": ""}} + } + _, e := q.control().UpdateOne(sc, bson.M{"_id": key}, update, options.Update().SetUpsert(true)) + return e + }) + return len(models), e + } + return 0, nil +} diff --git a/internal/store/query_replica_scale_local_test.go b/internal/store/query_replica_scale_local_test.go new file mode 100644 index 00000000..4d42399f --- /dev/null +++ b/internal/store/query_replica_scale_local_test.go @@ -0,0 +1,266 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" +) + +type replicaScalePhase struct { + Scope string `json:"scope"` + Actors int `json:"actors"` + Version string `json:"version"` + Calls int `json:"calls"` + Errors int `json:"errors"` + P95MS float64 `json:"p95Ms"` + SourceCommands int64 `json:"sourceCommands"` + SourceReplyBytes int64 `json:"sourceReplyBytes"` + SourceCPUSeconds float64 `json:"sourceCpuSeconds"` + Seconds float64 `json:"seconds"` +} + +func sourceProcessCPU(t *testing.T) float64 { + t.Helper() + if os.Getenv("QUERY_REPLICA_DOCKER_LAB") == "1" { + raw, e := exec.Command("docker", "compose", "-f", "../../scripts/local/query-replica.compose.yml", "exec", "-T", "source", "cat", "/sys/fs/cgroup/cpu.stat").Output() + if e != nil { + t.Fatal("Docker cgroup CPU unavailable", e) + } + for _, line := range strings.Split(string(raw), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "usage_usec" { + v, e := strconv.ParseFloat(fields[1], 64) + if e != nil { + t.Fatal(e) + } + return v / 1e6 + } + } + t.Fatal("Docker cgroup CPU counter missing") + } + b, e := os.ReadFile("../../local-data/query-replica-local/source/mongod.pid") + if e != nil { + t.Fatal(e) + } + pid := strings.TrimSpace(string(b)) + if _, e = strconv.Atoi(pid); e != nil { + t.Fatal(e) + } + raw, e := exec.Command("ps", "-p", pid, "-o", "time=").Output() + if e != nil { + t.Fatal(e) + } + parts := strings.Split(strings.TrimSpace(string(raw)), ":") + if len(parts) != 2 { + t.Fatal("unrecognized local process CPU format") + } + m, e := strconv.ParseFloat(parts[0], 64) + if e != nil { + t.Fatal(e) + } + sec, e := strconv.ParseFloat(parts[1], 64) + if e != nil { + t.Fatal(e) + } + return m*60 + sec +} + +// This is a fixed-data read-path comparison on two native local servers, NOT a +// production/4-core capacity claim. It includes incremental marker/ack traffic +// in the after measurement, and records process CPU, bytes and exact results. +func TestQueryReplicaScaleLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_SCALE_LOCAL") != "1" { + t.Skip("explicit isolated scale test required") + } + var measuring atomic.Bool + var commands, bytes atomic.Int64 + mon := &event.CommandMonitor{Started: func(ctx context.Context, e *event.CommandStartedEvent) { + if measuring.Load() { + commands.Add(1) + } + }, Succeeded: func(ctx context.Context, e *event.CommandSucceededEvent) { + if measuring.Load() { + bytes.Add(int64(len(e.Reply))) + } + }} + s, d, q := newLocalReplica(t, mon) + ctx := t.Context() + // Same metadata cardinality as the previously tested 26k-task fixture. Each + // item includes a large draft that must never cross to the query database. + const total = 26218 + for start := 0; start < total; start += 200 { + docs := []any{} + for i := start; i < start+200 && i < total; i++ { + docs = append(docs, bson.M{"id": fmt.Sprintf("item-%06d", i), "organizationId": "scale", "projectId": fmt.Sprintf("project-%02d", i%20), "pluginId": "annotation-qa", "batchId": "batch", "currentStageId": "quality_review", "status": "ready", "version": 1, "createdAt": "2026-11-01T00:00:00Z", "subject": bson.M{"itemKey": fmt.Sprintf("capture-%d/part0001", i), "payload": bson.M{"annotationDraft": strings.Repeat("draft", 256)}}}) + } + if _, e := s.humanItems.InsertMany(ctx, docs); e != nil { + t.Fatal(e) + } + } + for _, group := range []map[string][]mongo.IndexModel{PoolSearchReadIndexes(), PlatformReadIndexes()} { + for name, models := range group { + if _, e := s.db.Collection(name).Indexes().CreateMany(ctx, models); e != nil { + t.Fatal(e) + } + } + } + bootstrapAt := time.Now() + for n := 0; n < 200; n++ { + ready, e := q.RunOnce(ctx, true) + if e != nil { + t.Fatal(e) + } + if ready { + break + } + if n == 199 { + t.Fatal("bootstrap incomplete") + } + } + bootstrapSeconds := time.Since(bootstrapAt).Seconds() + sourceEnv, targetEnv := "native standalone; no CPU quota", "independent native replica set; no CPU quota" + if os.Getenv("QUERY_REPLICA_DOCKER_LAB") == "1" { + sourceEnv = "Docker standalone; 4 CPU/4GiB; WT1GiB" + targetEnv = "Docker replica set; 4 CPU/8GiB; WT2GiB" + } + nextVersion := 1 + var report []replicaScalePhase + complete := false + runRoot := filepath.Join("../../local-data/diagnostics/mongo-cpu/scale", time.Now().UTC().Format("20060102T150405.000000000Z")) + if e := os.MkdirAll(runRoot, 0700); e != nil { + t.Fatal(e) + } + t.Cleanup(func() { + artifact := map[string]any{"complete": complete && !t.Failed(), "kind": "bounded-local-read-path-comparison", "datasetTasks": total, "source": sourceEnv, "target": targetEnv, "productionCapacityAccepted": false, "bootstrapSeconds": bootstrapSeconds, "phases": report} + b, e := json.MarshalIndent(artifact, "", " ") + if e != nil { + t.Error(e) + return + } + if e = os.WriteFile(filepath.Join(runRoot, "report.json"), b, 0600); e != nil { + t.Error(e) + } + t.Log("performance evidence: " + runRoot) + }) + // Count candidates are exhaustive and page-independent. Scopes vary so the + // experiment cannot hide all work behind one global warm in-process cache. + for _, scope := range []string{"one-project", "all-projects"} { + for _, actors := range []int{81, 200, 500} { + for _, after := range []bool{false, true} { + if after { + if e := q.Acquire(ctx); e != nil { + t.Fatal(e) + } + if e := s.ConfigureQueryReplicaReports(d, time.Minute); e != nil { + t.Fatal(e) + } + s.EnableQueryReplicaPoolReads(true) + if e := q.PublishReady(ctx); e != nil { + t.Fatal(e) + } + } else { + s.queryReplicaReads = nil + } + version := "authority" + if after { + version = "query-replica" + } + commands.Store(0) + bytes.Store(0) + beforeCPU := sourceProcessCPU(t) + measuring.Store(true) + start := time.Now() + var wg sync.WaitGroup + durations := make([]float64, actors) + errs := make([]error, actors) + for a := 0; a < actors; a++ { + wg.Add(1) + go func(a int) { + defer wg.Done() + at := time.Now() + query := domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: fmt.Sprintf("project-%02d", a%20)} + if scope == "all-projects" { + query.ProjectID = "" + } + count := 0 + e := s.VisitHumanPoolCountCandidates(WithQueryReplicaCountRead(ctx), "scale", query, func(items []domain.HumanItemRecord) error { count += len(items); return nil }) + if e == nil { + want := total / 20 + if scope == "all-projects" { + want = total + } + if scope != "all-projects" && a%20 < total%20 { + want++ + } + if count != want { + e = fmt.Errorf("count differs: %d vs %d", count, want) + } + } + errs[a] = e + durations[a] = float64(time.Since(at).Microseconds()) / 1000 + }(a) + } + wg.Wait() + if after { + if e := q.Acquire(ctx); e != nil { + t.Fatal(e) + } + // A real semantic write and its projection/ack are charged to the new path. + _, e := s.humanItems.UpdateOne(ctx, bson.M{"id": "item-000000", "version": nextVersion}, bson.M{"$set": bson.M{"version": nextVersion + 1, "assignedUserId": fmt.Sprint("actor-", actors)}}) + if e != nil { + t.Fatal(e) + } + nextVersion++ + for _, name := range replicaCollections() { + if _, e = q.DrainStep(ctx, name); e != nil { + t.Fatal(e) + } + } + } + measuring.Store(false) + cpu := sourceProcessCPU(t) - beforeCPU + sort.Float64s(durations) + failures := 0 + for _, e := range errs { + if e != nil { + failures++ + t.Log(e) + } + } + report = append(report, replicaScalePhase{Actors: actors, Scope: scope, Version: version, Calls: actors, Errors: failures, P95MS: durations[(len(durations)-1)*95/100], SourceCommands: commands.Load(), SourceReplyBytes: bytes.Load(), SourceCPUSeconds: cpu, Seconds: time.Since(start).Seconds()}) + t.Logf("phase scope=%s actors=%d version=%s p95=%.1f errors=%d", scope, actors, version, report[len(report)-1].P95MS, failures) + } + } + } + artifact := map[string]any{"kind": "bounded-local-read-path-comparison", "datasetTasks": total, "source": sourceEnv, "target": targetEnv, "productionCapacityAccepted": false, "bootstrapSeconds": bootstrapSeconds, "phases": report} + root := "../../local-data/diagnostics/mongo-cpu" + b, e := json.MarshalIndent(artifact, "", " ") + if e != nil { + t.Fatal(e) + } + if e = os.WriteFile(filepath.Join(root, "scale-read-path.json"), b, 0600); e != nil { + t.Fatal(e) + } + t.Log(string(b)) + complete = true + for _, r := range report { + if r.Errors != 0 { + t.Errorf("%s/%d returned %d errors", r.Version, r.Actors, r.Errors) + } + } +} diff --git a/internal/store/query_replica_schema.go b/internal/store/query_replica_schema.go new file mode 100644 index 00000000..19d462a0 --- /dev/null +++ b/internal/store/query_replica_schema.go @@ -0,0 +1,157 @@ +package store + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + + "go.mongodb.org/mongo-driver/bson" +) + +const querySyncField = "_querySync" +const querySyncDeleteCollection = "querySyncDeleteIntents" + +// The replica contains query facts only. New query fields must be explicitly +// added here and verified against legacy results, never copied by exclusion. +var queryReplicaFields = map[string]string{ + "annotationIndexJobs": "id organizationId projectId runId status revalidateSource sourceFingerprint supersededSourceFingerprint createdAt", + "organizations": "id name slug status createdAt updatedAt", + "projects": "id organizationId name status createdAt updatedAt poolSearch", + "workflows": "id organizationId projectId name status active version createdAt updatedAt", + "runs": "id organizationId projectId workflowId taskId projectName workflowName trigger mode parentRunId status deliveryOutcome failure error createdAt endedAt archivedAt captureDirectories poolSearch inputs artifacts.id artifacts.name artifacts.path nodeExecutions.artifacts.id nodeExecutions.artifacts.name nodeExecutions.artifacts.path", + "captureRecords": "id organizationId projectId workflowId connectionId bucket prefix sourcePrefix captureId runId status businessStage discoveredAt completedAt poolSearch", + "annotationTasks": "id organizationId projectId workflowId dataTaskId runId supplierId assignedUserId status revision templateVersion annotationMode sourceIsFullVideo sourceDurationSeconds sourceVideo artifactId rightArtifactId createdAt updatedAt poolSearch", + "humanItems": "id organizationId projectId workflowId runId pluginId batchId gateNodeId legacyAuditTaskId currentRevision maxReworkRounds legacyAssignment pendingCombinedPayloadSha acceptanceSampleProbability acceptanceSampleValue acceptancePolicyRevision acceptedAt acceptedByUserId rejectedAt cancelledAt cancelledByUserId accountHandoff.pending reworkRoutingPending reviewOperationId legacyTaskId supplierId assignedUserId latestAuthorUserId processingScope processingProfile workgroupId vendorId currentStageId status version reviewRound claimedAt submittedAt createdAt updatedAt acceptanceClaim inheritedRework releasePartitionId pendingCombinedSubmissionId sourceItemId subject.itemKey subject.artifactId subject.mediaType annotationEpoch processingJob.sourceIndex processingJob.sourceFingerprint processingJob.error subject.payload.materialization.sourceIndex subject.payload.durationSeconds subject.payload.captureDirectory subject.payload.sourceBatchId subject.payload.strictThreeRole subject.payload.sliceWorkflow.status subject.payload.sliceWorkflow.phase subject.payload.items.materialization.coverage.durationSeconds processingJob.kind processingJob.status processingJob.stage processingJob.attempt processingJob.sliceCount processingJob.completedSliceCount poolSearch", + "humanBatches": "id organizationId projectId workflowId runId sourceBatchId gateNodeId pluginId supplierId status stages processingScope processingProfile requiredCapabilities requirePrePackageAcceptance createdAt updatedAt", + "humanReleasePartitions": "id organizationId projectId batchId sequence status uniqueKey itemIds parentRunId childRunId mode temporary createdAt updatedAt", + "annotationRevisions": "id organizationId projectId taskId revision authorUserId isRework countsAsOutput createdAt", + "sliceRevisions": "id organizationId projectId taskId revision authorUserId createdAt segments.start segments.end", + "annotationReviews": "id organizationId projectId taskId revision reviewRound stage decision reviewerUserId createdAt problemSliceIds", + "annotationWorklogEvents": "id organizationId projectId taskId userId supplierId eventType submissionId submissionKind outputDelta revision templateVersion durationSeconds videoSeconds createdAt mode temporary", + "humanEvents": "id organizationId itemId stageId eventType actorUserId round itemVersion createdAt payload.operationId payload.sourceItemIds payload.deliveryPackageId payload.reviewedRevision", + "reviewOperations": "_id operationId organizationId projectId userId itemId taskId stageId decision status sourceItemIds reviewerSupplierId createdAt completedAt effects.review.id effects.review.taskId effects.review.revision effects.beforeTask.id effects.beforeTask.supplierId effects.worklog.id effects.worklog.supplierId reviewRound reviewVideoSources draft.modelVersion draft.problems.id draft.problems.scope draft.problems.sourceItemId draft.problems.sliceId draft.problems.actionId draft.problems.start draft.problems.end draft.problems.timeBasis effects.beforeTask.projectId effects.beforeTask.templateVersion effects.beforeItem.id effects.beforeItem.legacyTaskId effects.beforeItem.latestAuthorUserId effects.beforeItem.acceptedByUserId effects.beforeItem.supplierId effects.beforeItem.projectId effects.beforeItem.currentRevision effects.beforeItem.subject.start effects.beforeItem.subject.end effects.item.id effects.item.legacyTaskId effects.item.latestAuthorUserId effects.item.supplierId effects.item.currentRevision effects.item.acceptedByUserId", + "reworkBaselines": "_id organizationId projectId itemId round stageId reviewOperationId problems.id problems.scope problems.sourceItemId problems.origin.stageId createdAt", + "reworkHandlings": "_id organizationId projectId itemId round operationId skippedOpinionIds updatedAt", + "deliveryPackages": "id organizationId projectId parentRunId childRunId supplierId itemIds status manifestDigest releasePartitionId acceptanceHumanItemId mode temporary finalExportedAt qualitySummary.prePackageAcceptance acceptanceReviews.id acceptanceReviews.decision acceptanceReviews.reviewerUserId acceptanceReviews.createdAt", + "users": "id organizationId name role supplierId status createdAt updatedAt", + "suppliers": "id organizationId name status createdAt updatedAt", + "productionStatsScopes": "_id projectIds revision updatedAt updatedBy", +} + +func queryReplicaProjection(collection string) bson.M { + p := bson.M{"_id": 1, querySyncField: 1} + for _, field := range strings.Fields(queryReplicaFields[collection]) { + p[field] = 1 + } + return p +} +func queryFieldValue(v any, path []string) any { + if len(path) == 0 { + return v + } + switch m := v.(type) { + case bson.M: + return queryFieldValue(m[path[0]], path[1:]) + case map[string]any: + return queryFieldValue(m[path[0]], path[1:]) + case bson.A: + out := make([]any, 0, len(m)) + for _, item := range m { + out = append(out, queryFieldValue(item, path)) + } + return out + case []any: + out := make([]any, 0, len(m)) + for _, item := range m { + out = append(out, queryFieldValue(item, path)) + } + return out + } + return nil +} +func queryReplicaSignature(collection string, d bson.M) string { + values := bson.M{} + if collection == "humanItems" && d["pendingCombinedSubmissionId"] != nil && d["pendingCombinedSubmissionId"] != "" { + values["version"] = d["version"] + } + for _, field := range strings.Fields(queryReplicaFields[collection]) { + // Current edit versions are checked against authority on page hydration. + // Draft-only saves must not create another synchronization write cycle. + if field == "_id" || field == "version" || field == "updatedAt" { + continue + } + if field == "poolSearch" { + for _, p := range []string{"poolSearch.text", "poolSearch.payloadDirectories", "poolSearch.durationSeconds", "poolSearch.searchText", "poolSearch.textVersion", "poolSearch.schema"} { + values[p] = queryFieldValue(d, strings.Split(p, ".")) + } + continue + } + values[field] = queryFieldValue(d, strings.Split(field, ".")) + } + b, err := json.Marshal(values) + if err != nil { + return "" + } + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +func queryUpdateRelevant(collection string, update bson.M) bool { + relevant := func(path string) bool { + if collection == "humanItems" && path == "version" { + return true + } + parts := strings.Split(path, ".") + normal := parts[:0] + for _, part := range parts { + if part == "" { + continue + } + numeric := true + for _, r := range part { + if r < '0' || r > '9' { + numeric = false + break + } + } + if numeric || strings.HasPrefix(part, "$") { + continue + } + normal = append(normal, part) + } + path = strings.Join(normal, ".") + for _, field := range strings.Fields(queryReplicaFields[collection]) { + if field == "_id" || field == "version" || field == "updatedAt" { + continue + } + candidates := []string{field} + if field == "poolSearch" { + candidates = []string{"poolSearch.text", "poolSearch.payloadDirectories", "poolSearch.durationSeconds", "poolSearch.searchText", "poolSearch.textVersion", "poolSearch.schema"} + } + for _, f := range candidates { + if path == f || strings.HasPrefix(path, f+".") || strings.HasPrefix(f, path+".") { + return true + } + } + } + return false + } + for op, value := range update { + fields, e := queryDocument(value) + if e != nil { + return true + } + for key, v := range fields { + if relevant(key) { + return true + } + if op == "$rename" { + if path, ok := v.(string); ok && relevant(path) { + return true + } + } + } + } + return false +} diff --git a/internal/store/query_replica_valkey_local_test.go b/internal/store/query_replica_valkey_local_test.go new file mode 100644 index 00000000..0a2699b0 --- /dev/null +++ b/internal/store/query_replica_valkey_local_test.go @@ -0,0 +1,97 @@ +package store + +import ( + "context" + "os" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/queryserving" + "go.mongodb.org/mongo-driver/bson" +) + +func TestQueryReplicaSharedValkeyAcrossReadersAndOutageLocal(t *testing.T) { + if os.Getenv("QUERY_REPLICA_VALKEY_LOCAL_TEST") != "1" { + t.Skip("owned Valkey lab required") + } + s, target, q := newLocalReplica(t) + ctx := t.Context() + cache, e := queryserving.NewManagedValkey("redis://127.0.0.1:27139", domain.NewID("query-replica-cache-test")) + if e != nil { + t.Fatal(e) + } + defer cache.Close() + ready := false + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + cache.Put(ctx, "probe", []byte("ready"), time.Second) + if b, ok := cache.Get(ctx, "probe"); ok && string(b) == "ready" { + ready = true + break + } + time.Sleep(25 * time.Millisecond) + } + if !ready { + t.Fatal("owned Valkey did not become usable") + } + if _, e = s.humanItems.InsertOne(ctx, bson.M{"id": "i", "organizationId": "o", "pluginId": "annotation-qa", "status": "ready", "acceptanceClaim": bson.M{"userId": "owner", "supplierId": "supplier"}, "subject": bson.M{"payload": bson.M{"annotationDraft": "never replicate"}}}); e != nil { + t.Fatal(e) + } + finishLocalReplica(t, q) + target.ConfigureQueryServing(cache, nil) + if e = s.ConfigureQueryReplicaReports(target, time.Minute); e != nil { + t.Fatal(e) + } + s.EnableQueryReplicaPoolReads(true) + query := domain.HumanItemQuery{PluginID: "annotation-qa"} + count := func(source *MongoStore) error { + n := 0 + e := source.VisitHumanPoolCountCandidates(WithQueryReplicaCountRead(ctx), "o", query, func(rows []domain.HumanItemRecord) error { + n += len(rows) + if len(rows) > 0 && (rows[0].AcceptanceClaim == nil || rows[0].AcceptanceClaim.UserID != "owner" || rows[0].Subject.Payload != nil) { + t.Error("shared cache changed private metadata") + } + return nil + }) + if e == nil && n != 1 { + t.Errorf("count=%d", n) + } + return e + } + if e = count(s); e != nil { + t.Fatal(e) + } + before := cache.Statistics()["hits"] + // A second API process has no in-process candidate entries, but shares the + // exact query database/epoch and namespace, so it must use Valkey's record. + source2, target2 := NewMongoStore(s.db), NewMongoStore(target.db) + target2.ConfigureQueryServing(cache, nil) + if e = source2.ConfigureQueryReplicaReports(target2, time.Minute); e != nil { + t.Fatal(e) + } + source2.EnableQueryReplicaPoolReads(true) + if e = count(source2); e != nil { + t.Fatal(e) + } + if cache.Statistics()["hits"] <= before { + t.Fatal("second reader did not use shared cache") + } + // Unavailable shared cache must use query facts, never authority history. + cache.Close() + source3, target3 := NewMongoStore(s.db), NewMongoStore(target.db) + target3.ConfigureQueryServing(cache, nil) + if e = source3.ConfigureQueryReplicaReports(target3, time.Minute); e != nil { + t.Fatal(e) + } + source3.EnableQueryReplicaPoolReads(true) + if e = count(source3); e != nil { + t.Fatal(e) + } + // Context cancellation does not convert cache failure into a successful zero. + canceled, cancel := context.WithCancel(ctx) + cancel() + if e = source3.VisitHumanPoolCountCandidates(WithQueryReplicaCountRead(canceled), "o", query, func([]domain.HumanItemRecord) error { return nil }); e == nil { + t.Fatal("cancelled read succeeded") + } +} diff --git a/internal/store/query_source_changes.go b/internal/store/query_source_changes.go index 160aba16..3f180aaf 100644 --- a/internal/store/query_source_changes.go +++ b/internal/store/query_source_changes.go @@ -58,7 +58,7 @@ func querySourceHash(doc bson.M) querySourceDigest { } func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error { for collection := range querySearchSources { - cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}}) + cur, err := b.store.queryCollection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$project", Value: querySearchSourceProjection(collection)}}}) if err != nil { return err } @@ -81,7 +81,7 @@ func (b *QueryReadBuilder) seedSourceDigests(ctx context.Context) error { func (b *QueryReadBuilder) searchSourceChanged(ctx context.Context, collection string, key bson.M) (bool, string, error) { identity := querySourceIdentity(collection, key["_id"]) old, exists := b.sourceDigests[identity] - cur, err := b.store.db.Collection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}}) + cur, err := b.store.queryCollection(collection).Aggregate(ctx, mongo.Pipeline{{{Key: "$match", Value: key}}, {{Key: "$project", Value: querySearchSourceProjection(collection)}}}) if err != nil { return false, "", err } @@ -127,7 +127,7 @@ func (b *QueryReadBuilder) observeQuerySourceEvent(ctx context.Context, l queryM var doc struct { Org string `bson:"organizationId"` } - if err := b.store.db.Collection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments { + if err := b.store.queryCollection(collection).FindOne(ctx, key, options.FindOne().SetProjection(bson.M{"organizationId": 1})).Decode(&doc); err != nil && err != mongo.ErrNoDocuments { return err } org = doc.Org diff --git a/internal/store/query_sync_bulk.go b/internal/store/query_sync_bulk.go new file mode 100644 index 00000000..a4f65a48 --- /dev/null +++ b/internal/store/query_sync_bulk.go @@ -0,0 +1,136 @@ +package store + +import ( + "context" + "errors" + "reflect" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func (c *queryCollection) taggedModel(model mongo.WriteModel) (mongo.WriteModel, error) { + if model == nil || (reflect.ValueOf(model).Kind() == reflect.Pointer && reflect.ValueOf(model).IsNil()) { + return nil, errors.New("bulk model is nil") + } + switch m := model.(type) { + case *mongo.InsertOneModel: + copy := *m + v, e := c.taggedInsert(m.Document) + copy.Document = v + return ©, e + case *mongo.UpdateOneModel: + copy := *m + v, e := c.taggedUpdate(m.Update) + copy.Update = v + return ©, e + case *mongo.UpdateManyModel: + copy := *m + v, e := c.taggedUpdate(m.Update) + copy.Update = v + return ©, e + case *mongo.ReplaceOneModel: + p, e := c.replacementPipeline(m.Replacement) + copy := mongo.NewUpdateOneModel().SetFilter(m.Filter).SetUpdate(p) + copy.Upsert = m.Upsert + copy.Collation = m.Collation + copy.Hint = m.Hint + return copy, e + default: + return model, nil + } +} +func (c *queryCollection) BulkWrite(ctx context.Context, models []mongo.WriteModel, opts ...*options.BulkWriteOptions) (*mongo.BulkWriteResult, error) { + if !c.tracks() { + return c.Collection.BulkWrite(ctx, models, opts...) + } + deletes := false + for _, m := range models { + if m == nil || (reflect.ValueOf(m).Kind() == reflect.Pointer && reflect.ValueOf(m).IsNil()) { + return nil, errors.New("bulk model is nil") + } + switch m.(type) { + case *mongo.DeleteOneModel, *mongo.DeleteManyModel: + deletes = true + } + } + if !deletes { + copy := make([]mongo.WriteModel, len(models)) + for i, m := range models { + v, e := c.taggedModel(m) + if e != nil { + return nil, e + } + copy[i] = v + } + return c.Collection.BulkWrite(ctx, copy, opts...) + } + ordered := true + for _, v := range opts { + if v != nil && v.Ordered != nil { + ordered = *v.Ordered + } + } + result := &mongo.BulkWriteResult{UpsertedIDs: map[int64]any{}} + var failed mongo.BulkWriteException + for i, m := range models { + var err error + switch v := m.(type) { + case *mongo.DeleteOneModel: + o := options.Delete() + o.Collation = v.Collation + o.Hint = v.Hint + r, e := c.DeleteOne(ctx, v.Filter, o) + err = e + if r != nil { + result.DeletedCount += r.DeletedCount + } + case *mongo.DeleteManyModel: + o := options.Delete() + o.Collation = v.Collation + o.Hint = v.Hint + r, e := c.DeleteMany(ctx, v.Filter, o) + err = e + if r != nil { + result.DeletedCount += r.DeletedCount + } + default: + tagged, e := c.taggedModel(m) + err = e + if e == nil { + r, e := c.Collection.BulkWrite(ctx, []mongo.WriteModel{tagged}, opts...) + err = e + if r != nil { + result.InsertedCount += r.InsertedCount + result.MatchedCount += r.MatchedCount + result.ModifiedCount += r.ModifiedCount + result.DeletedCount += r.DeletedCount + result.UpsertedCount += r.UpsertedCount + for _, id := range r.UpsertedIDs { + result.UpsertedIDs[int64(i)] = id + } + } + } + } + if err != nil { + var bulk mongo.BulkWriteException + if !errors.As(err, &bulk) { + return result, err + } + for _, write := range bulk.WriteErrors { + write.Index = i + failed.WriteErrors = append(failed.WriteErrors, write) + } + if bulk.WriteConcernError != nil { + failed.WriteConcernError = bulk.WriteConcernError + } + if ordered { + break + } + } + } + if len(failed.WriteErrors) > 0 || failed.WriteConcernError != nil { + return result, failed + } + return result, nil +} diff --git a/internal/store/query_sync_delete.go b/internal/store/query_sync_delete.go new file mode 100644 index 00000000..1d3c9f90 --- /dev/null +++ b/internal/store/query_sync_delete.go @@ -0,0 +1,217 @@ +package store + +import ( + "context" + "errors" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const queryDeleteLease = 30 * time.Second + +var errQueryDeleteFenced = errors.New("query deletion intent fenced") + +type queryDeleteIntent struct { + ID string `bson:"_id"` + Audit bool `bson:"audit,omitempty"` + Collection string `bson:"collection"` + Phase string `bson:"phase"` + LeaseUntil time.Time `bson:"leaseUntil"` + CreatedAt time.Time `bson:"createdAt"` +} + +func (c *queryCollection) deleteIntents() *mongo.Collection { + return c.Database().Collection(querySyncDeleteCollection) +} +func (c *queryCollection) beginDelete(ctx context.Context) (queryDeleteIntent, func(), error) { + now := time.Now().UTC() + i := queryDeleteIntent{ID: domain.NewID("query-delete"), Collection: c.Name(), Phase: "prepared", LeaseUntil: now.Add(queryDeleteLease), CreatedAt: now} + if _, e := c.deleteIntents().InsertOne(ctx, i); e != nil { + return i, func() {}, e + } + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(queryDeleteLease / 3) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + renew, cancel := context.WithTimeout(context.Background(), 2*time.Second) + _, _ = c.deleteIntents().UpdateOne(renew, bson.M{"_id": i.ID, "phase": bson.M{"$in": bson.A{"prepared", "deleting"}}}, bson.M{"$set": bson.M{"leaseUntil": time.Now().UTC().Add(queryDeleteLease)}}) + cancel() + } + } + }() + return i, func() { close(stop); <-done }, nil +} +func (c *queryCollection) deletePhase(ctx context.Context, id, from, to string) error { + r, e := c.deleteIntents().UpdateOne(ctx, bson.M{"_id": id, "phase": from}, bson.M{"$set": bson.M{"phase": to}}) + if e != nil { + return e + } + if r.MatchedCount != 1 { + return errQueryDeleteFenced + } + return nil +} +func deleteOptionsRead(opts []*options.DeleteOptions, sorts ...any) *options.FindOneAndUpdateOptions { + o := options.FindOneAndUpdate().SetProjection(bson.M{"_id": 1}).SetReturnDocument(options.After) + for _, v := range opts { + if v == nil { + continue + } + o.Collation = v.Collation + o.Hint = v.Hint + o.Let = v.Let + o.Comment = v.Comment + } + if len(sorts) > 0 && sorts[0] != nil { + o.SetSort(sorts[0]) + } + return o +} +func (c *queryCollection) prepareDelete(ctx context.Context, filter any, many bool, opts []*options.DeleteOptions, sorts ...any) (queryDeleteIntent, any, func(), error) { + i, finish, e := c.beginDelete(ctx) + if e != nil { + return i, nil, finish, e + } + claim := bson.M{"$set": bson.M{querySyncField + ".deleteClaim": i.ID}} + available := bson.M{"$and": bson.A{filter, bson.M{querySyncField + ".deleteClaim": bson.M{"$exists": false}}}} + constrained := bson.M{"$and": bson.A{filter, bson.M{querySyncField + ".deleteClaim": i.ID}}} + if many { + o := options.Update() + for _, v := range opts { + if v == nil { + continue + } + o.Collation = v.Collation + o.Hint = v.Hint + o.Let = v.Let + o.Comment = v.Comment + } + if _, e = c.Collection.UpdateMany(ctx, available, claim, o); e != nil { + return i, nil, finish, e + } + } else { + var selected bson.M + e = c.Collection.FindOneAndUpdate(ctx, available, claim, deleteOptionsRead(opts, sorts...)).Decode(&selected) + if errors.Is(e, mongo.ErrNoDocuments) { + err := c.Collection.FindOne(ctx, filter, options.FindOne().SetProjection(bson.M{"_id": 1})).Err() + if err == nil { + return i, nil, finish, errQueryDeleteFenced + } + if !errors.Is(err, mongo.ErrNoDocuments) { + return i, nil, finish, err + } + _ = c.deletePhase(ctx, i.ID, "prepared", "complete") + return i, constrained, finish, nil + } + if e != nil { + return i, nil, finish, e + } + constrained["$and"] = bson.A{filter, bson.M{"_id": selected["_id"], querySyncField + ".deleteClaim": i.ID}} + } + if many { + err := c.Collection.FindOne(ctx, bson.M{"$and": bson.A{filter, bson.M{querySyncField + ".deleteClaim": bson.M{"$ne": i.ID}}}}, options.FindOne().SetProjection(bson.M{"_id": 1})).Err() + if err == nil { + return i, nil, finish, errQueryDeleteFenced + } + if !errors.Is(err, mongo.ErrNoDocuments) { + return i, nil, finish, err + } + } + if e = c.deletePhase(ctx, i.ID, "prepared", "deleting"); e != nil { + return i, nil, finish, e + } + return i, constrained, finish, nil +} +func (c *queryCollection) DeleteOne(ctx context.Context, f any, o ...*options.DeleteOptions) (*mongo.DeleteResult, error) { + if !c.tracks() { + return c.Collection.DeleteOne(ctx, f, o...) + } + i, filter, finish, e := c.prepareDelete(ctx, f, false, o) + defer finish() + if e != nil { + return nil, e + } + r, e := c.Collection.DeleteOne(ctx, filter, o...) + if e == nil { + _ = c.deletePhase(ctx, i.ID, "deleting", "complete") + } + return r, e +} +func (c *queryCollection) DeleteMany(ctx context.Context, f any, o ...*options.DeleteOptions) (*mongo.DeleteResult, error) { + if !c.tracks() { + return c.Collection.DeleteMany(ctx, f, o...) + } + i, filter, finish, e := c.prepareDelete(ctx, f, true, o) + defer finish() + if e != nil { + return nil, e + } + r, e := c.Collection.DeleteMany(ctx, filter, o...) + if e == nil { + _ = c.deletePhase(ctx, i.ID, "deleting", "complete") + } + return r, e +} +func (c *queryCollection) FindOneAndDelete(ctx context.Context, f any, o ...*options.FindOneAndDeleteOptions) *mongo.SingleResult { + if !c.tracks() { + return c.Collection.FindOneAndDelete(ctx, f, o...) + } + del := options.Delete() + var sort any + for _, v := range o { + if v == nil { + continue + } + sort = v.Sort + del.Collation = v.Collation + del.Hint = v.Hint + del.Let = v.Let + del.Comment = v.Comment + } + i, filter, finish, e := c.prepareDelete(ctx, f, false, []*options.DeleteOptions{del}, sort) + defer finish() + if e != nil { + return mongo.NewSingleResultFromDocument(nil, e, nil) + } + r := c.Collection.FindOneAndDelete(ctx, filter, o...) + if r.Err() == nil || errors.Is(r.Err(), mongo.ErrNoDocuments) { + _ = c.deletePhase(ctx, i.ID, "deleting", "complete") + } + return r +} + +// A crashed prepared operation cannot start deleting after this CAS. For an +// already deleting operation, clearing its per-record claims fences any late +// delete command before reconciliation. No annotation/state is restored. +func (c *queryCollection) recoverDelete(ctx context.Context, i queryDeleteIntent, now time.Time) (bool, error) { + if i.Phase == "prepared" || i.Phase == "deleting" { + if i.LeaseUntil.After(now) { + return false, nil + } + r, e := c.deleteIntents().UpdateOne(ctx, bson.M{"_id": i.ID, "phase": i.Phase, "leaseUntil": bson.M{"$lte": now}}, bson.M{"$set": bson.M{"phase": "cancelled"}}) + if e != nil { + return false, e + } + if r.MatchedCount != 1 { + return false, nil + } + } else if i.Phase != "complete" && i.Phase != "cancelled" && i.Phase != "reconcile" { + return false, nil + } + if _, e := c.Collection.UpdateMany(ctx, bson.M{querySyncField + ".deleteClaim": i.ID}, bson.M{"$unset": bson.M{querySyncField + ".deleteClaim": ""}}); e != nil { + return false, e + } + _, e := c.deleteIntents().UpdateOne(ctx, bson.M{"_id": i.ID, "phase": bson.M{"$in": bson.A{"complete", "cancelled", "reconcile"}}}, bson.M{"$set": bson.M{"phase": "reconcile"}}) + return e == nil, e +} diff --git a/internal/store/query_sync_local_test.go b/internal/store/query_sync_local_test.go new file mode 100644 index 00000000..35641e1e --- /dev/null +++ b/internal/store/query_sync_local_test.go @@ -0,0 +1,294 @@ +package store + +import ( + "context" + "os" + "reflect" + "testing" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func newQuerySyncSource(t *testing.T, monitors ...*event.CommandMonitor) *MongoStore { + t.Helper() + if os.Getenv("QUERY_REPLICA_LOCAL_TEST") != "1" { + t.Skip("isolated source/query Mongo required") + } + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + opts := options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true") + if len(monitors) > 0 { + opts.SetMonitor(monitors[0]) + } + c, e := mongo.Connect(ctx, opts) + if e != nil { + t.Fatal(e) + } + var hello bson.M + if e = c.Database("admin").RunCommand(ctx, bson.M{"hello": 1}).Decode(&hello); e != nil { + t.Fatal(e) + } + if hello["setName"] != nil { + t.Fatal("source must reproduce production's non-CDC standalone entry point") + } + db := c.Database(domain.NewID("query_sync_test")) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = db.Drop(ctx) + _ = c.Disconnect(ctx) + }) + s := NewMongoStore(db) + s.querySyncEnabled.Store(true) + return s +} +func readQuerySync(t *testing.T, c mongoCollection, id any) (bson.M, bson.M) { + t.Helper() + var d bson.M + if e := c.FindOne(t.Context(), bson.M{"_id": id}).Decode(&d); e != nil { + t.Fatal(e) + } + return d, deletionDoc(d[querySyncField]) +} +func querySyncRev(d bson.M) int64 { return int64(supplierNumber(d["revision"])) } +func TestQuerySyncAtomicVersionAndDraftSuppressionLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + id := primitive.NewObjectID() + c := s.queryCollection("humanItems") + doc := bson.M{"_id": id, "id": "item", "organizationId": "org", "projectId": "p", "version": 1, "status": "claimed", "currentStageId": "annotation_combined", "subject": bson.M{"itemKey": "capture/part0001", "payload": bson.M{"annotationDraft": bson.M{"description": "original annotation"}}}} + if _, e := c.InsertOne(ctx, doc); e != nil { + t.Fatal(e) + } + _, meta := readQuerySync(t, c, id) + if querySyncRev(meta) != 1 || meta["dirty"] != true { + t.Fatal(meta) + } + _, e := c.Collection.UpdateOne(ctx, bson.M{"_id": id}, bson.M{"$set": bson.M{querySyncField + ".dirty": false}}) + if e != nil { + t.Fatal(e) + } + delete(doc, "_id") + doc["version"] = 2 + doc["subject"] = bson.M{"itemKey": "capture/part0001", "payload": bson.M{"annotationDraft": bson.M{"description": "latest annotation"}}} + r, e := c.ReplaceOne(ctx, bson.M{"id": "item", "version": 1}, doc) + if e != nil || r.MatchedCount != 1 { + t.Fatal(r, e) + } + current, meta := readQuerySync(t, c, id) + if querySyncRev(meta) != 1 || meta["dirty"] != false || deletionString(deletionDoc(deletionDoc(deletionDoc(current["subject"])["payload"])["annotationDraft"]), "description") != "latest annotation" { + t.Fatal("draft edit or source identity changed incorrectly", meta) + } + r, e = c.ReplaceOne(ctx, bson.M{"id": "item", "version": 1}, doc) + if e != nil || r.MatchedCount != 0 { + t.Fatal("business version CAS bypassed", r, e) + } + doc["status"] = "submitted" + doc["version"] = 3 + if _, e = c.ReplaceOne(ctx, bson.M{"id": "item", "version": 2}, doc); e != nil { + t.Fatal(e) + } + _, meta = readQuerySync(t, c, id) + if querySyncRev(meta) != 2 || meta["dirty"] != true { + t.Fatal("semantic change lost", meta) + } +} +func TestQuerySyncIrrelevantLogWritesAndUpsertIdentityLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + c := s.queryCollection("runs") + doc := bson.M{"id": "r", "organizationId": "org", "status": "running", "nodeExecutions": bson.A{bson.M{"nodeId": "n", "logs": bson.A{}}}} + if _, e := c.UpdateOne(ctx, bson.M{"id": "r"}, bson.M{"$set": doc}, options.Update().SetUpsert(true)); e != nil { + t.Fatal(e) + } + var inserted bson.M + if e := c.FindOne(ctx, bson.M{"id": "r"}).Decode(&inserted); e != nil { + t.Fatal(e) + } + if inserted["_id"] == nil { + t.Fatal("upsert lost generated identity") + } + id := inserted["_id"] + _, meta := readQuerySync(t, c, id) + before := querySyncRev(meta) + if _, e := c.UpdateOne(ctx, bson.M{"id": "r"}, bson.M{"$push": bson.M{"nodeExecutions.0.logs": bson.M{"message": "log must not be replicated"}}}); e != nil { + t.Fatal(e) + } + _, meta = readQuerySync(t, c, id) + if querySyncRev(meta) != before { + t.Fatal("logs caused projection cycle", meta) + } +} +func TestQuerySyncDeleteRecoveryFencesLateDeleteAndPreservesEditsLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + c := s.queryCollection("humanItems") + id := primitive.NewObjectID() + if _, e := c.InsertOne(ctx, bson.M{"_id": id, "id": "item", "version": 1, "subject": bson.M{"payload": bson.M{"annotationDraft": "must remain"}}}); e != nil { + t.Fatal(e) + } + i, filter, finish, e := c.prepareDelete(ctx, bson.M{"id": "item", "version": 1}, false, nil) + defer finish() + if e != nil { + t.Fatal(e) + } + expired := time.Now().Add(-time.Second) + _, e = c.deleteIntents().UpdateOne(ctx, bson.M{"_id": i.ID}, bson.M{"$set": bson.M{"leaseUntil": expired}}) + if e != nil { + t.Fatal(e) + } + i.Phase = "deleting" + i.LeaseUntil = expired + if ok, e := c.recoverDelete(ctx, i, time.Now()); e != nil || !ok { + t.Fatal(ok, e) + } + if r, e := c.Collection.DeleteOne(ctx, filter); e != nil || r.DeletedCount != 0 { + t.Fatal("late delete passed cancelled claim", r, e) + } + doc, _ := readQuerySync(t, c, id) + if deletionString(deletionDoc(deletionDoc(doc["subject"])["payload"]), "annotationDraft") != "must remain" { + t.Fatal("recovery changed annotation") + } + if _, e = c.DeleteOne(ctx, bson.M{"id": "item", "version": 1}); e != nil { + t.Fatal(e) + } + if n, e := c.CountDocuments(ctx, bson.M{"id": "item"}); e != nil || n != 0 { + t.Fatal("actual deletion changed", n, e) + } +} + +func TestQuerySyncBulkPipelineAndDeleteSortLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + c := s.queryCollection("humanItems") + if _, e := c.BulkWrite(ctx, []mongo.WriteModel{mongo.NewInsertOneModel().SetDocument(bson.M{"id": "a", "status": "ready", "version": 1}), mongo.NewInsertOneModel().SetDocument(bson.M{"id": "b", "status": "ready", "version": 1})}); e != nil { + t.Fatal(e) + } + for _, update := range []any{bson.D{{Key: "$set", Value: bson.M{"status": "claimed"}}}, bson.A{bson.M{"$set": bson.M{"status": "submitted"}}}, mongo.Pipeline{{{Key: "$set", Value: bson.M{"status": "ready"}}}}} { + if _, e := c.UpdateOne(ctx, bson.M{"id": "a"}, update); e != nil { + t.Fatal(e) + } + } + var deleted bson.M + if e := c.FindOneAndDelete(ctx, bson.M{}, options.FindOneAndDelete().SetSort(bson.D{{Key: "id", Value: -1}})).Decode(&deleted); e != nil || deleted["id"] != "b" { + t.Fatal("delete sort changed", deleted, e) + } + var nilModel *mongo.UpdateOneModel + if _, e := c.BulkWrite(ctx, []mongo.WriteModel{nilModel}); e == nil { + t.Fatal("nil model not rejected") + } + if _, e := c.UpdateOne(ctx, bson.M{"id": "a"}, mongo.Pipeline{}); e == nil { + t.Fatal("empty pipeline unexpectedly updates marker") + } +} +func TestQuerySyncPendingSubmissionVersionIsQueryRelevantLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + c := s.queryCollection("humanItems") + d := bson.M{"id": "i", "version": 2, "status": "claimed", "pendingCombinedSubmissionId": "fanyu-v5-combined-i-v1"} + r, e := c.InsertOne(ctx, d) + if e != nil { + t.Fatal(e) + } + _, before := readQuerySync(t, c, r.InsertedID) + d["version"] = 3 + if _, e = c.ReplaceOne(ctx, bson.M{"id": "i", "version": 2}, d); e != nil { + t.Fatal(e) + } + _, after := readQuerySync(t, c, r.InsertedID) + if querySyncRev(after) != querySyncRev(before)+1 { + t.Fatal("submission conflict classification version missed") + } +} + +func TestQuerySyncImmutableReceiptRetryDoesNotWriteLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + c := s.queryCollection("annotationWorklogEvents") + update := bson.M{"$setOnInsert": bson.M{"id": "receipt", "taskId": "t", "userId": "u", "eventType": "combined_submitted"}} + if _, e := c.UpdateOne(ctx, bson.M{"id": "receipt"}, update, options.Update().SetUpsert(true)); e != nil { + t.Fatal(e) + } + var first bson.M + if e := c.FindOne(ctx, bson.M{"id": "receipt"}).Decode(&first); e != nil { + t.Fatal(e) + } + result, e := c.UpdateOne(ctx, bson.M{"id": "receipt"}, update, options.Update().SetUpsert(true)) + if e != nil || result.ModifiedCount != 0 || result.UpsertedCount != 0 { + t.Fatal("immutable retry produced a write", result, e) + } + var after bson.M + if e = c.FindOne(ctx, bson.M{"id": "receipt"}).Decode(&after); e != nil { + t.Fatal(e) + } + if !reflect.DeepEqual(first, after) { + t.Fatal("receipt retry changed data or sync marker") + } +} +func TestQuerySyncSubmissionRecoveryEventsRemainIdempotentLocal(t *testing.T) { + s := newQuerySyncSource(t) + if e := s.EnsureIndexes(t.Context()); e != nil { + t.Fatal(e) + } + checkSubmissionRecoveryEventOnce(t, s) +} + +func TestQuerySyncAuthorityContractsLocal(t *testing.T) { + for _, tc := range []struct { + name string + run func(*testing.T, Store) + }{ + {"supplier-management", supplierManagementContract}, + {"supplier-deletion", supplierCascadeContract}, + {"acceptance-authorization", testSharedAcceptanceStore}, + {"acceptance-revocation", testAcceptanceRevocationRaceAndQueue}, + {"progress-reads", testLatestProgressReads}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newQuerySyncSource(t) + if e := s.EnsureIndexes(t.Context()); e != nil { + t.Fatal(e) + } + tc.run(t, queryContractStore{s}) + }) + } +} + +func TestQuerySyncReplacementOptionsPreserveSDKSemanticsLocal(t *testing.T) { + s := newQuerySyncSource(t) + ctx := t.Context() + r, e := s.humanItems.ReplaceOne(ctx, bson.M{"id": "new"}, bson.M{"id": "new", "version": 1}, options.Replace().SetUpsert(true), options.Replace().SetComment("second option")) + if e != nil || r.UpsertedCount != 1 { + t.Fatal("multiple option upsert lost", r, e) + } + var after bson.M + e = s.humanItems.FindOneAndReplace(ctx, bson.M{"id": "new"}, bson.M{"id": "new", "version": 2}, options.FindOneAndReplace().SetReturnDocument(options.After), options.FindOneAndReplace().SetComment("second option")).Decode(&after) + if e != nil || supplierNumber(after["version"]) != 2 { + t.Fatal("multiple option return document lost", after, e) + } +} + +// Older store contracts leave the source key empty. Supply a distinct synthetic +// identity so those fixtures also satisfy the production unique index; retain +// every real authorization, CAS, deletion and annotation assertion. +type queryContractStore struct{ *MongoStore } + +func (s queryContractStore) UpsertHumanItem(ctx context.Context, item domain.HumanItemRecord) error { + if item.Subject.ItemKey == "" { + item.Subject.ItemKey = item.ID + } + return s.MongoStore.UpsertHumanItem(ctx, item) +} + +func (s queryContractStore) ClaimHumanItemVersioned(ctx context.Context, item domain.HumanItemRecord, version int, role domain.UserRole) (bool, error) { + if item.Subject.ItemKey == "" { + item.Subject.ItemKey = item.ID + } + return s.MongoStore.ClaimHumanItemVersioned(ctx, item, version, role) +} diff --git a/internal/store/query_tracked_collection.go b/internal/store/query_tracked_collection.go new file mode 100644 index 00000000..06ae1e15 --- /dev/null +++ b/internal/store/query_tracked_collection.go @@ -0,0 +1,261 @@ +package store + +import ( + "context" + "errors" + "reflect" + "strings" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func (s *MongoStore) queryCollection(name string, opts ...*options.CollectionOptions) *queryCollection { + return &queryCollection{Collection: s.db.Collection(name, opts...), store: s} +} + +type queryCollection struct { + *mongo.Collection + store *MongoStore +} + +func (c *queryCollection) tracks() bool { + return c.store != nil && c.store.querySyncEnabled.Load() && queryReplicaFields[c.Name()] != "" +} +func queryDocument(value any) (bson.M, error) { + b, e := bson.Marshal(value) + if e != nil { + return nil, e + } + var d bson.M + e = bson.Unmarshal(b, &d) + return d, e +} +func queryMetadataStage(signature, mode string) bson.D { + changed := any(true) + if signature != "" { + changed = bson.M{"$or": bson.A{bson.M{"$ne": bson.A{bson.M{"$ifNull": bson.A{"$_querySync.signature", ""}}, signature}}, bson.M{"$ne": bson.A{bson.M{"$ifNull": bson.A{"$_querySync.mode", ""}}, mode}}}} + } + metadata := bson.M{ + "revision": bson.M{"$add": bson.A{bson.M{"$ifNull": bson.A{"$_querySync.revision", 0}}, bson.M{"$cond": bson.A{changed, 1, 0}}}}, + "dirty": bson.M{"$cond": bson.A{changed, true, bson.M{"$ifNull": bson.A{"$_querySync.dirty", false}}}}, + "signature": signature, "mode": mode, "producer": "query-source/v1", + } + return bson.D{{Key: "$set", Value: bson.M{querySyncField: bson.M{"$mergeObjects": bson.A{bson.M{"$ifNull": bson.A{"$_querySync", bson.M{}}}, metadata}}}}} +} +func (c *queryCollection) taggedInsert(value any) (any, error) { + d, e := queryDocument(value) + if e != nil { + return nil, e + } + d[querySyncField] = bson.M{"revision": int64(1), "dirty": true, "signature": queryReplicaSignature(c.Name(), d), "mode": "replace", "producer": "query-source/v1"} + return d, nil +} +func (c *queryCollection) taggedUpdate(value any) (any, error) { + if value == nil { + return nil, errors.New("query update is nil") + } + rv := reflect.ValueOf(value) + _, orderedDocument := value.(bson.D) + if rv.Kind() == reflect.Slice && !orderedDocument { + p := mongo.Pipeline{} + for i := 0; i < rv.Len(); i++ { + raw, e := bson.Marshal(rv.Index(i).Interface()) + if e != nil { + return nil, e + } + var stage bson.D + if e = bson.Unmarshal(raw, &stage); e != nil { + return nil, e + } + p = append(p, stage) + } + if len(p) == 0 { + return nil, errors.New("update pipeline is empty") + } + return append(p, queryMetadataStage("", "unknown")), nil + } + + d, e := queryDocument(value) + if e != nil { + return nil, e + } + if len(d) == 1 { + if insert, ok := d["$setOnInsert"]; ok { + tagged, e := c.taggedInsert(insert) + if e != nil { + return nil, e + } + return bson.M{"$setOnInsert": tagged}, nil + } + } + if !queryUpdateRelevant(c.Name(), d) { + return value, nil + } + set, hasSet := d["$set"] + if len(d) == 1 && hasSet { + fields, e := queryDocument(set) + if e != nil { + return nil, e + } + _, full := fields["id"] + for field := range fields { + if strings.Contains(field, ".") || strings.Contains(field, "$") { + full = false + } + } + if full { + delete(fields, querySyncField) + literal := bson.M{} + for k, v := range fields { + literal[k] = bson.M{"$literal": v} + } + return mongo.Pipeline{{{Key: "$set", Value: literal}}, queryMetadataStage(queryReplicaSignature(c.Name(), fields), "set")}, nil + } + } + // Keep operator semantics, array filters and existing version conditions. + // Unknown/partial mutations conservatively mark one small atomic field set. + inc := bson.M{} + if v, ok := d["$inc"]; ok { + inc, e = queryDocument(v) + if e != nil { + return nil, e + } + } + fields := bson.M{} + if hasSet { + fields, e = queryDocument(set) + if e != nil { + return nil, e + } + } + for k := range fields { + if strings.HasPrefix(k, querySyncField) { + delete(fields, k) + } + } + fields[querySyncField+".dirty"] = true + fields[querySyncField+".signature"] = "" + fields[querySyncField+".mode"] = "unknown" + fields[querySyncField+".producer"] = "query-source/v1" + inc[querySyncField+".revision"] = 1 + d["$set"] = fields + d["$inc"] = inc + return d, nil +} +func (c *queryCollection) InsertOne(ctx context.Context, d any, o ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) { + if c.tracks() { + var e error + d, e = c.taggedInsert(d) + if e != nil { + return nil, e + } + } + return c.Collection.InsertOne(ctx, d, o...) +} +func (c *queryCollection) InsertMany(ctx context.Context, docs []any, o ...*options.InsertManyOptions) (*mongo.InsertManyResult, error) { + if c.tracks() { + copy := make([]any, len(docs)) + for n, d := range docs { + var e error + copy[n], e = c.taggedInsert(d) + if e != nil { + return nil, e + } + } + docs = copy + } + return c.Collection.InsertMany(ctx, docs, o...) +} +func (c *queryCollection) UpdateOne(ctx context.Context, f, u any, o ...*options.UpdateOptions) (*mongo.UpdateResult, error) { + if c.tracks() { + var e error + u, e = c.taggedUpdate(u) + if e != nil { + return nil, e + } + } + return c.Collection.UpdateOne(ctx, f, u, o...) +} +func (c *queryCollection) UpdateMany(ctx context.Context, f, u any, o ...*options.UpdateOptions) (*mongo.UpdateResult, error) { + if c.tracks() { + var e error + u, e = c.taggedUpdate(u) + if e != nil { + return nil, e + } + } + return c.Collection.UpdateMany(ctx, f, u, o...) +} +func (c *queryCollection) FindOneAndUpdate(ctx context.Context, f, u any, o ...*options.FindOneAndUpdateOptions) *mongo.SingleResult { + if c.tracks() { + var e error + u, e = c.taggedUpdate(u) + if e != nil { + return mongo.NewSingleResultFromDocument(nil, e, nil) + } + } + return c.Collection.FindOneAndUpdate(ctx, f, u, o...) +} +func (c *queryCollection) ReplaceOne(ctx context.Context, f, replacement any, opts ...*options.ReplaceOptions) (*mongo.UpdateResult, error) { + if !c.tracks() { + return c.Collection.ReplaceOne(ctx, f, replacement, opts...) + } + p, e := c.replacementPipeline(replacement) + if e != nil { + return nil, e + } + o := options.Update() + v := options.MergeReplaceOptions(opts...) + o.Upsert = v.Upsert + o.Collation = v.Collation + o.Hint = v.Hint + o.BypassDocumentValidation = v.BypassDocumentValidation + o.Comment = v.Comment + o.Let = v.Let + return c.Collection.UpdateOne(ctx, f, p, o) +} + +func (c *queryCollection) replacementPipeline(replacement any) (mongo.Pipeline, error) { + d, e := queryDocument(replacement) + if e != nil { + return nil, e + } + delete(d, querySyncField) + // Preserve Mongo replacement semantics (including omitted-field removal) and + // the existing version predicate. Only the private marker is carried forward. + expression := bson.M{"$mergeObjects": bson.A{bson.M{"_id": "$_id", querySyncField: "$_querySync"}, bson.M{"$literal": d}}} + p := mongo.Pipeline{{{Key: "$replaceWith", Value: expression}}, queryMetadataStage(queryReplicaSignature(c.Name(), d), "replace")} + return p, nil +} + +func (c *queryCollection) FindOneAndReplace(ctx context.Context, f, replacement any, opts ...*options.FindOneAndReplaceOptions) *mongo.SingleResult { + if !c.tracks() { + return c.Collection.FindOneAndReplace(ctx, f, replacement, opts...) + } + p, e := c.replacementPipeline(replacement) + if e != nil { + return mongo.NewSingleResultFromDocument(nil, e, nil) + } + o := options.FindOneAndUpdate() + v := options.MergeFindOneAndReplaceOptions(opts...) + o.Upsert = v.Upsert + o.Collation = v.Collation + o.Hint = v.Hint + o.BypassDocumentValidation = v.BypassDocumentValidation + o.Comment = v.Comment + o.Let = v.Let + o.MaxTime = v.MaxTime + o.Projection = v.Projection + o.ReturnDocument = v.ReturnDocument + o.Sort = v.Sort + return c.Collection.FindOneAndUpdate(ctx, f, p, o) +} + +func (c *queryCollection) Drop(ctx context.Context) error { + if c.tracks() { + return ErrQueryReplicaUnsupported + } + return c.Collection.Drop(ctx) +} diff --git a/internal/store/review_continuation.go b/internal/store/review_continuation.go index f51b285f..772f167c 100644 --- a/internal/store/review_continuation.go +++ b/internal/store/review_continuation.go @@ -76,7 +76,7 @@ func (s *MemoryStore) CompareAndSwapReviewContinuation(_ context.Context, v doma func (s *MongoStore) readReviewContinuation(ctx context.Context, id string) (*domain.ReviewContinuation, error) { var v domain.ReviewContinuation - e := s.db.Collection("reviewContinuations").FindOne(ctx, bson.M{"_id": id}).Decode(&v) + e := s.queryCollection("reviewContinuations").FindOne(ctx, bson.M{"_id": id}).Decode(&v) if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil } @@ -88,7 +88,7 @@ func (s *MongoStore) readReviewContinuation(ctx context.Context, id string) (*do func (s *MongoStore) writeReviewContinuation(ctx context.Context, v domain.ReviewContinuation, expected int) (bool, error) { v.Revision = expected + 1 - c := s.db.Collection("reviewContinuations") + c := s.queryCollection("reviewContinuations") if expected == 0 { _, e := c.InsertOne(ctx, v) if mongo.IsDuplicateKeyError(e) { @@ -120,7 +120,7 @@ func (s *MongoStore) applyReviewContinuationAdmission(ctx context.Context, sessi applied = current != nil && current.AdmissionID == admission.ID } // Matching the receipt ID preserves any later admission and any page change. - _, err = s.db.Collection("reviewSessions").UpdateOne(ctx, + _, err = s.queryCollection("reviewSessions").UpdateOne(ctx, bson.M{"_id": sessionID, "continuationAdmission.id": admission.ID}, bson.M{"$unset": bson.M{"continuationAdmission": ""}, "$inc": bson.M{"revision": 1}}) return applied, err @@ -131,9 +131,9 @@ func (s *MongoStore) GetReviewContinuation(ctx context.Context, id string) (*dom // next ordinary queue request, even if the source page has already changed. filter := bson.M{"continuationAdmission.continuationId": id} read := options.Find().SetHint(reviewAdmissionLookupIndex) - cursor, err := s.db.Collection("reviewSessions").Find(ctx, filter, read) + cursor, err := s.queryCollection("reviewSessions").Find(ctx, filter, read) if poolSearchHintMissing(err) && ctx.Err() == nil { - cursor, err = s.db.Collection("reviewSessions").Find(ctx, filter, read.SetHint(nil)) + cursor, err = s.queryCollection("reviewSessions").Find(ctx, filter, read.SetHint(nil)) } if err != nil { return nil, err diff --git a/internal/store/review_decision.go b/internal/store/review_decision.go index fe3e29cc..855c8503 100644 --- a/internal/store/review_decision.go +++ b/internal/store/review_decision.go @@ -52,7 +52,7 @@ func (s *MemoryStore) CompareAndSwapReviewOperation(_ context.Context, v domain. } func (s *MongoStore) GetReviewOperation(ctx context.Context, id string) (*domain.ReviewOperation, error) { var v domain.ReviewOperation - e := s.db.Collection("reviewOperations").FindOne(ctx, bson.M{"_id": id}).Decode(&v) + e := s.queryCollection("reviewOperations").FindOne(ctx, bson.M{"_id": id}).Decode(&v) if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil } @@ -65,9 +65,9 @@ func (s *MongoStore) FindReviewOperation(ctx context.Context, org, user, resourc var v domain.ReviewOperation filter := bson.M{"organizationId": org, "userId": user, "operationId": operation, "$or": bson.A{bson.M{"itemId": resource}, bson.M{"taskId": resource}}} read := options.FindOne().SetHint(reviewOperationLookupIndex) - e := s.db.Collection("reviewOperations").FindOne(ctx, filter, read).Decode(&v) + e := s.queryCollection("reviewOperations").FindOne(ctx, filter, read).Decode(&v) if poolSearchHintMissing(e) && ctx.Err() == nil { - e = s.db.Collection("reviewOperations").FindOne(ctx, filter, read.SetHint(nil)).Decode(&v) + e = s.queryCollection("reviewOperations").FindOne(ctx, filter, read.SetHint(nil)).Decode(&v) } if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil @@ -79,7 +79,7 @@ func (s *MongoStore) FindReviewOperation(ctx context.Context, org, user, resourc } func (s *MongoStore) CompareAndSwapReviewOperation(ctx context.Context, v domain.ReviewOperation, expected int) (bool, error) { v.Revision = expected + 1 - c := s.db.Collection("reviewOperations") + c := s.queryCollection("reviewOperations") if expected == 0 { _, e := c.InsertOne(ctx, v) if mongo.IsDuplicateKeyError(e) { diff --git a/internal/store/review_draft.go b/internal/store/review_draft.go index a4dd1217..8be8e664 100644 --- a/internal/store/review_draft.go +++ b/internal/store/review_draft.go @@ -35,7 +35,7 @@ func (s *MemoryStore) CompareAndSwapReviewDraft(_ context.Context, next domain.R } func (s *MongoStore) GetReviewDraft(ctx context.Context, key string) (*domain.ReviewDraft, error) { var v domain.ReviewDraft - e := s.db.Collection("reviewDrafts").FindOne(ctx, bson.M{"_id": key}).Decode(&v) + e := s.queryCollection("reviewDrafts").FindOne(ctx, bson.M{"_id": key}).Decode(&v) if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil } @@ -46,7 +46,7 @@ func (s *MongoStore) GetReviewDraft(ctx context.Context, key string) (*domain.Re } func (s *MongoStore) CompareAndSwapReviewDraft(ctx context.Context, next domain.ReviewDraft, expected int) (bool, error) { next.Version = expected + 1 - c := s.db.Collection("reviewDrafts") + c := s.queryCollection("reviewDrafts") if expected == 0 { _, e := c.InsertOne(ctx, next) if mongo.IsDuplicateKeyError(e) { @@ -84,7 +84,7 @@ func (s *MemoryStore) GetReviewDraftArchive(_ context.Context, key string) (*dom return &v, nil } func (s *MongoStore) ArchiveReviewDraft(ctx context.Context, draft domain.ReviewDraft) error { - _, err := s.db.Collection("reviewDraftArchives").InsertOne(ctx, draft) + _, err := s.queryCollection("reviewDraftArchives").InsertOne(ctx, draft) if mongo.IsDuplicateKeyError(err) { return nil } @@ -92,7 +92,7 @@ func (s *MongoStore) ArchiveReviewDraft(ctx context.Context, draft domain.Review } func (s *MongoStore) GetReviewDraftArchive(ctx context.Context, key string) (*domain.ReviewDraft, error) { var draft domain.ReviewDraft - err := s.db.Collection("reviewDraftArchives").FindOne(ctx, bson.M{"_id": key}).Decode(&draft) + err := s.queryCollection("reviewDraftArchives").FindOne(ctx, bson.M{"_id": key}).Decode(&draft) if errors.Is(err, mongo.ErrNoDocuments) { return nil, nil } diff --git a/internal/store/review_session.go b/internal/store/review_session.go index ea65a0c3..b35b5d6d 100644 --- a/internal/store/review_session.go +++ b/internal/store/review_session.go @@ -33,7 +33,7 @@ func (s *MemoryStore) CompareAndSwapReviewSession(_ context.Context, next domain } func (s *MongoStore) GetReviewSession(ctx context.Context, key string) (*domain.ReviewSession, error) { var v domain.ReviewSession - e := s.db.Collection("reviewSessions").FindOne(ctx, bson.M{"_id": key}).Decode(&v) + e := s.queryCollection("reviewSessions").FindOne(ctx, bson.M{"_id": key}).Decode(&v) if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil } @@ -44,7 +44,7 @@ func (s *MongoStore) GetReviewSession(ctx context.Context, key string) (*domain. } func (s *MongoStore) CompareAndSwapReviewSession(ctx context.Context, next domain.ReviewSession, expected int) (bool, error) { next.Revision = expected + 1 - c := s.db.Collection("reviewSessions") + c := s.queryCollection("reviewSessions") if expected == 0 { _, e := c.InsertOne(ctx, next) if mongo.IsDuplicateKeyError(e) { diff --git a/internal/store/rework_baseline.go b/internal/store/rework_baseline.go index 02490919..3a462f91 100644 --- a/internal/store/rework_baseline.go +++ b/internal/store/rework_baseline.go @@ -42,7 +42,7 @@ func (s *MemoryStore) UpsertReworkBaseline(_ context.Context, record domain.Rewo func (s *MongoStore) GetReworkBaseline(ctx context.Context, id string) (*domain.ReworkBaselineRecord, error) { var v domain.ReworkBaselineRecord - e := s.db.Collection("reworkBaselines").FindOne(ctx, bson.M{"_id": id}).Decode(&v) + e := s.queryCollection("reworkBaselines").FindOne(ctx, bson.M{"_id": id}).Decode(&v) if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil } @@ -53,7 +53,7 @@ func (s *MongoStore) GetReworkBaseline(ctx context.Context, id string) (*domain. } func (s *MongoStore) UpsertReworkBaseline(ctx context.Context, record domain.ReworkBaselineRecord) error { - _, e := s.db.Collection("reworkBaselines").ReplaceOne(ctx, bson.M{"_id": record.ID}, record, options.Replace().SetUpsert(true)) + _, e := s.queryCollection("reworkBaselines").ReplaceOne(ctx, bson.M{"_id": record.ID}, record, options.Replace().SetUpsert(true)) return e } @@ -89,7 +89,7 @@ func (s *MemoryStore) CompareAndSwapReworkHandling(_ context.Context, next domai func (s *MongoStore) GetReworkHandling(ctx context.Context, id string) (*domain.ReworkHandlingRecord, error) { var v domain.ReworkHandlingRecord - e := s.db.Collection("reworkHandlings").FindOne(ctx, bson.M{"_id": id}).Decode(&v) + e := s.queryCollection("reworkHandlings").FindOne(ctx, bson.M{"_id": id}).Decode(&v) if errors.Is(e, mongo.ErrNoDocuments) { return nil, nil } @@ -101,7 +101,7 @@ func (s *MongoStore) GetReworkHandling(ctx context.Context, id string) (*domain. func (s *MongoStore) CompareAndSwapReworkHandling(ctx context.Context, next domain.ReworkHandlingRecord, expected int) (bool, error) { if expected == 0 { - result, e := s.db.Collection("reworkHandlings").InsertOne(ctx, bson.M{ + result, e := s.queryCollection("reworkHandlings").InsertOne(ctx, bson.M{ "_id": next.ID, "organizationId": next.OrganizationID, "itemId": next.ItemID, "decisionOperationId": next.DecisionOperationID, "operationId": next.OperationID, "actorUserId": next.ActorUserID, "round": next.Round, "skippedOpinionIds": next.SkippedOpinionIDs, "version": 1, "updatedAt": next.UpdatedAt, }) @@ -115,7 +115,7 @@ func (s *MongoStore) CompareAndSwapReworkHandling(ctx context.Context, next doma } filter := bson.M{"_id": next.ID, "version": expected} next.Version = expected + 1 - result, e := s.db.Collection("reworkHandlings").UpdateOne(ctx, filter, bson.M{"$set": bson.M{ + result, e := s.queryCollection("reworkHandlings").UpdateOne(ctx, filter, bson.M{"$set": bson.M{ "decisionOperationId": next.DecisionOperationID, "operationId": next.OperationID, "actorUserId": next.ActorUserID, "skippedOpinionIds": next.SkippedOpinionIDs, "updatedAt": next.UpdatedAt, "version": next.Version, }}) if e != nil { diff --git a/internal/store/supplier_deletion.go b/internal/store/supplier_deletion.go index a5d2abab..d68d4b19 100644 --- a/internal/store/supplier_deletion.go +++ b/internal/store/supplier_deletion.go @@ -120,7 +120,7 @@ func (s *MongoStore) loadSupplierSnapshot(ctx context.Context, org string) (supp rows = append(rows, supplierDoc(r)) } } else { - cursor, err := s.db.Collection(col).Find(ctx, orgCompatibilityFilter(org), options.Find().SetProjection(supplierDeletionProjection(col)).SetBatchSize(5000).SetLimit(50001)) + cursor, err := s.queryCollection(col).Find(ctx, orgCompatibilityFilter(org), options.Find().SetProjection(supplierDeletionProjection(col)).SetBatchSize(5000).SetLimit(50001)) if err != nil { return err } @@ -185,7 +185,7 @@ func (s *MemoryStore) GetSupplierDeletion(_ context.Context, id string) (*domain } func (s *MongoStore) GetSupplierDeletion(ctx context.Context, id string) (*domain.SupplierDeletionTask, error) { var t domain.SupplierDeletionTask - e := s.db.Collection("supplierDeletionTasks").FindOne(ctx, bson.M{"_id": id}).Decode(&t) + e := s.queryCollection("supplierDeletionTasks").FindOne(ctx, bson.M{"_id": id}).Decode(&t) if e == mongo.ErrNoDocuments { return nil, nil } @@ -204,7 +204,7 @@ func (s *MemoryStore) ListSupplierDeletions(_ context.Context, org string) ([]do return out, nil } func (s *MongoStore) ListSupplierDeletions(ctx context.Context, org string) ([]domain.SupplierDeletionTask, error) { - cur, e := s.db.Collection("supplierDeletionTasks").Find(ctx, bson.M{"organizationId": org}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(100)) + cur, e := s.queryCollection("supplierDeletionTasks").Find(ctx, bson.M{"organizationId": org}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(100)) if e != nil { return nil, e } @@ -273,7 +273,7 @@ func (s *MongoStore) BeginSupplierDeletion(ctx context.Context, t domain.Supplie if len(fresh.Blockers) > 0 { return nil, fmt.Errorf("删除范围存在阻止项") } - _, e = s.db.Collection("supplierDeletionTasks").InsertOne(ctx, t) + _, e = s.queryCollection("supplierDeletionTasks").InsertOne(ctx, t) if e != nil { return nil, e } @@ -325,7 +325,7 @@ func (s *MemoryStore) ClaimSupplierDeletion(_ context.Context, id, owner, now, u } func (s *MongoStore) ClaimSupplierDeletion(ctx context.Context, id, owner, now, until string) (*domain.SupplierDeletionTask, bool, error) { var t domain.SupplierDeletionTask - e := s.db.Collection("supplierDeletionTasks").FindOneAndUpdate(ctx, bson.M{"_id": id, "status": bson.M{"$ne": "completed"}, "$or": bson.A{bson.M{"leaseOwner": ""}, bson.M{"leaseOwner": nil}, bson.M{"leaseUntil": bson.M{"$lte": now}}}}, bson.M{"$set": bson.M{"status": "running", "error": "", "leaseOwner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetReturnDocument(options.After)).Decode(&t) + e := s.queryCollection("supplierDeletionTasks").FindOneAndUpdate(ctx, bson.M{"_id": id, "status": bson.M{"$ne": "completed"}, "$or": bson.A{bson.M{"leaseOwner": ""}, bson.M{"leaseOwner": nil}, bson.M{"leaseUntil": bson.M{"$lte": now}}}}, bson.M{"$set": bson.M{"status": "running", "error": "", "leaseOwner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetReturnDocument(options.After)).Decode(&t) if e == mongo.ErrNoDocuments { x, e := s.GetSupplierDeletion(ctx, id) return x, false, e @@ -352,7 +352,7 @@ func (s *MongoStore) SaveSupplierDeletion(ctx context.Context, t domain.Supplier fields["leaseOwner"] = "" fields["leaseUntil"] = "" } - r, e := s.db.Collection("supplierDeletionTasks").UpdateOne(ctx, bson.M{"_id": t.ID, "leaseOwner": owner}, bson.M{"$set": fields}) + r, e := s.queryCollection("supplierDeletionTasks").UpdateOne(ctx, bson.M{"_id": t.ID, "leaseOwner": owner}, bson.M{"$set": fields}) return e == nil && r.MatchedCount == 1, e } func (s *MemoryStore) RenewSupplierDeletion(_ context.Context, id, owner, until string) (bool, error) { @@ -367,7 +367,7 @@ func (s *MemoryStore) RenewSupplierDeletion(_ context.Context, id, owner, until return true, nil } func (s *MongoStore) RenewSupplierDeletion(ctx context.Context, id, owner, until string) (bool, error) { - r, e := s.db.Collection("supplierDeletionTasks").UpdateOne(ctx, bson.M{"_id": id, "leaseOwner": owner, "status": "running"}, bson.M{"$set": bson.M{"leaseUntil": until}}) + r, e := s.queryCollection("supplierDeletionTasks").UpdateOne(ctx, bson.M{"_id": id, "leaseOwner": owner, "status": "running"}, bson.M{"$set": bson.M{"leaseUntil": until}}) return e == nil && r.MatchedCount == 1, e } func supplierLeaseValid(t *domain.SupplierDeletionTask, owner string) bool { diff --git a/internal/store/supplier_deletion_guard.go b/internal/store/supplier_deletion_guard.go index a2cd5976..19fdcfe6 100644 --- a/internal/store/supplier_deletion_guard.go +++ b/internal/store/supplier_deletion_guard.go @@ -86,7 +86,7 @@ func (s *MongoStore) supplierRecordJobs(ctx context.Context, col, id string, d b } filter = bson.M{"$or": scoped} } - cursor, e := s.db.Collection("supplierDeletionTasks").Find(ctx, filter) + cursor, e := s.queryCollection("supplierDeletionTasks").Find(ctx, filter) if e != nil { return nil, e } @@ -167,7 +167,7 @@ func scrubSupplierDocument(d bson.M, p domain.SupplierDeletionPlan) bool { } return changed } -func supplierGuardedMongoWrite(ctx context.Context, col *mongo.Collection, id string, item any, write func(context.Context, bson.M) error) error { +func supplierGuardedMongoWrite(ctx context.Context, col mongoCollection, id string, item any, write func(context.Context, bson.M) error) error { d := supplierDoc(item) writeSource := write write = func(ctx context.Context, document bson.M) error { diff --git a/internal/store/supplier_deletion_records.go b/internal/store/supplier_deletion_records.go index 94aae710..6905f09c 100644 --- a/internal/store/supplier_deletion_records.go +++ b/internal/store/supplier_deletion_records.go @@ -79,7 +79,7 @@ func (s *MongoStore) ExecuteSupplierDeletionRecords(ctx context.Context, t domai for col, ids := range t.Plan.SharedRecords { for _, id := range ids { var before bson.M - e := s.db.Collection(col).FindOne(ctx, bson.M{"id": id, "organizationId": t.OrganizationID}).Decode(&before) + e := s.queryCollection(col).FindOne(ctx, bson.M{"id": id, "organizationId": t.OrganizationID}).Decode(&before) if e == mongo.ErrNoDocuments { continue } @@ -118,7 +118,7 @@ func (s *MongoStore) ExecuteSupplierDeletionRecords(ctx context.Context, t domai if _, ok := before["version"]; ok { update["$inc"] = bson.M{"version": 1} } - r, e := s.db.Collection(col).UpdateOne(ctx, filter, update) + r, e := s.queryCollection(col).UpdateOne(ctx, filter, update) if e != nil { return nil, e } @@ -140,7 +140,7 @@ func (s *MongoStore) ExecuteSupplierDeletionRecords(ctx context.Context, t domai if len(ids) == 0 { continue } - if _, e := s.db.Collection(col).DeleteMany(ctx, bson.M{"id": bson.M{"$in": ids}, "organizationId": t.OrganizationID}); e != nil { + if _, e := s.queryCollection(col).DeleteMany(ctx, bson.M{"id": bson.M{"$in": ids}, "organizationId": t.OrganizationID}); e != nil { return nil, e } } @@ -207,7 +207,7 @@ func (s *MongoStore) CancelSupplierQueuedProcessing(ctx context.Context, p domai } } if len(p.Records["annotationIndexJobs"]) > 0 { - _, e := s.db.Collection("annotationIndexJobs").UpdateMany(ctx, bson.M{"id": bson.M{"$in": p.Records["annotationIndexJobs"]}, "organizationId": p.OrganizationID, "status": "queued"}, bson.M{"$set": bson.M{"status": "failed", "error": "供应商正在删除"}}) + _, e := s.queryCollection("annotationIndexJobs").UpdateMany(ctx, bson.M{"id": bson.M{"$in": p.Records["annotationIndexJobs"]}, "organizationId": p.OrganizationID, "status": "queued"}, bson.M{"$set": bson.M{"status": "failed", "error": "供应商正在删除"}}) if e != nil { return e } @@ -229,6 +229,6 @@ func (s *MongoStore) SupplierIndexingActive(ctx context.Context, p domain.Suppli if len(p.Records["annotationIndexJobs"]) == 0 { return false, nil } - n, e := s.db.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"id": bson.M{"$in": p.Records["annotationIndexJobs"]}, "organizationId": p.OrganizationID, "status": "running"}) + n, e := s.queryCollection("annotationIndexJobs").CountDocuments(ctx, bson.M{"id": bson.M{"$in": p.Records["annotationIndexJobs"]}, "organizationId": p.OrganizationID, "status": "running"}) return n > 0, e } diff --git a/internal/store/supplier_management.go b/internal/store/supplier_management.go index d4ec348c..5e5734ff 100644 --- a/internal/store/supplier_management.go +++ b/internal/store/supplier_management.go @@ -122,7 +122,7 @@ func workflowUsesSupplier(w domain.WorkflowRecord, id string) bool { } func (s *MongoStore) MutateSupplier(ctx context.Context, org, id, name string, remove bool) (*domain.SupplierRecord, error) { - guards := s.db.Collection("account_mutation_guards") + guards := s.queryCollection("account_mutation_guards") key := "suppliers:" + normalizedOrganizationID(org) _, err := guards.UpdateOne(ctx, bson.M{"_id": key}, bson.M{"$setOnInsert": bson.M{"version": 0}}, options.Update().SetUpsert(true)) if err != nil && !mongo.IsDuplicateKeyError(err) { @@ -155,7 +155,7 @@ func (s *MongoStore) MutateSupplier(ctx context.Context, org, id, name string, r return nil, e } refs := map[string]int64{} - for label, col := range map[string]*mongo.Collection{"账号": s.users, "项目成员": s.projectMembers, "项目工作流": s.projectWorkflowInstances, "标注任务": s.annotationTasks, "人工任务": s.humanItems, "人工批次": s.humanBatches, "交付记录": s.deliveryPackages, "作业记录": s.annotationWorklogEvents, "历史成员": s.annotationMemberships} { + for label, col := range map[string]mongoCollection{"账号": s.users, "项目成员": s.projectMembers, "项目工作流": s.projectWorkflowInstances, "标注任务": s.annotationTasks, "人工任务": s.humanItems, "人工批次": s.humanBatches, "交付记录": s.deliveryPackages, "作业记录": s.annotationWorklogEvents, "历史成员": s.annotationMemberships} { n, e := col.CountDocuments(sc, bson.M{"$or": bson.A{bson.M{"supplierId": id}, bson.M{"vendorId": id}}}) if e != nil { return nil, e diff --git a/internal/store/supplier_pool_conversion.go b/internal/store/supplier_pool_conversion.go index 1b77f0df..e8c8b2e5 100644 --- a/internal/store/supplier_pool_conversion.go +++ b/internal/store/supplier_pool_conversion.go @@ -436,14 +436,14 @@ func (s *MongoStore) supplierPoolEvidence(ctx context.Context, org, project stri } row.HasReview = count > 0 } - for collection, target := range map[*mongo.Collection]*bool{s.humanRejects: &row.HasReject, s.db.Collection("reviewDrafts"): &row.HasReviewDraft} { + for collection, target := range map[mongoCollection]*bool{s.humanRejects: &row.HasReject, s.queryCollection("reviewDrafts"): &row.HasReviewDraft} { count, e := collection.CountDocuments(ctx, bson.M{"organizationId": org, "itemId": item.ID}, nil) if e != nil { return nil, e } *target = count > 0 } - count, err := s.db.Collection("humanClaimRequests").CountDocuments(ctx, bson.M{"organizationId": org, "itemId": item.ID}) + count, err := s.queryCollection("humanClaimRequests").CountDocuments(ctx, bson.M{"organizationId": org, "itemId": item.ID}) if err != nil { return nil, err } @@ -455,7 +455,7 @@ func (s *MongoStore) supplierPoolEvidence(ctx context.Context, org, project stri func (s *MongoStore) PreviewSupplierPoolConversion(ctx context.Context, org, project, actor string) (SupplierPoolPreview, error) { var pending supplierPoolManifest - err := s.db.Collection("supplierPoolPreviews").FindOne(ctx, bson.M{"organizationId": org, "projectId": project, "state": "pending"}).Decode(&pending) + err := s.queryCollection("supplierPoolPreviews").FindOne(ctx, bson.M{"organizationId": org, "projectId": project, "state": "pending"}).Decode(&pending) if err == nil { if pending.ActorUserID != actor { return SupplierPoolPreview{}, ErrSupplierPoolConversionPending @@ -477,11 +477,11 @@ func (s *MongoStore) PreviewSupplierPoolConversion(ctx context.Context, org, pro } preview := buildSupplierPoolPreview(project, rows) manifest := supplierPoolManifestForPreview(org, project, actor, preview, rows) - _, err = s.db.Collection("supplierPoolPreviews").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "expiresAt", Value: 1}}, Options: options.Index().SetExpireAfterSeconds(0)}) + _, err = s.queryCollection("supplierPoolPreviews").Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "expiresAt", Value: 1}}, Options: options.Index().SetExpireAfterSeconds(0)}) if err != nil { return SupplierPoolPreview{}, err } - if _, err := s.db.Collection("supplierPoolPreviews").InsertOne(ctx, manifest); err != nil { + if _, err := s.queryCollection("supplierPoolPreviews").InsertOne(ctx, manifest); err != nil { return SupplierPoolPreview{}, err } preview.PreviewToken = manifest.ID @@ -491,7 +491,7 @@ func (s *MongoStore) PreviewSupplierPoolConversion(ctx context.Context, org, pro // Claims and conversion take the same project lock. A pending journal also // blocks a claim after a process restart, until the original token is retried. func (s *MongoStore) supplierPoolConversionPending(ctx context.Context, batchID string) (bool, error) { - count, err := s.db.Collection("supplierPoolPreviews").CountDocuments(ctx, bson.M{"state": "pending", "batchIds": batchID}) + count, err := s.queryCollection("supplierPoolPreviews").CountDocuments(ctx, bson.M{"state": "pending", "batchIds": batchID}) return count > 0, err } @@ -510,14 +510,14 @@ func supplierPoolOldTimeFilter(value string) any { return value } -func (s *MongoStore) supplierPoolMarker(ctx context.Context, col *mongo.Collection, id, operationID string) (bool, error) { +func (s *MongoStore) supplierPoolMarker(ctx context.Context, col mongoCollection, id, operationID string) (bool, error) { count, err := col.CountDocuments(ctx, bson.M{"id": id, "poolConversionOperationId": operationID, "supplierId": bson.M{"$in": bson.A{"", nil}}}) return count == 1, err } func (s *MongoStore) convertSupplierPoolBatch(ctx context.Context, manifest supplierPoolManifest, snaps []supplierPoolSnapshot, operationID string) error { first := snaps[0] - ensure := func(col *mongo.Collection, id string, filter bson.M, update bson.M) error { + ensure := func(col mongoCollection, id string, filter bson.M, update bson.M) error { marked, err := s.supplierPoolMarker(ctx, col, id, operationID) if err != nil { return err @@ -586,7 +586,7 @@ func (s *MongoStore) ConvertSupplierPool(ctx context.Context, org, project, prev return SupplierPoolConversionResult{}, ErrSupplierPoolPreviewChanged } value, err := s.withSupplierProjectLocks(ctx, []string{project}, func(ctx context.Context) (any, error) { - col := s.db.Collection("supplierPoolPreviews") + col := s.queryCollection("supplierPoolPreviews") var manifest supplierPoolManifest if err := col.FindOne(ctx, bson.M{"_id": previewToken, "organizationId": org, "projectId": project, "actorUserId": actor}).Decode(&manifest); err != nil { if errors.Is(err, mongo.ErrNoDocuments) { diff --git a/internal/store/supplier_pool_conversion_mongo_test.go b/internal/store/supplier_pool_conversion_mongo_test.go index 2a263893..e5de1f30 100644 --- a/internal/store/supplier_pool_conversion_mongo_test.go +++ b/internal/store/supplier_pool_conversion_mongo_test.go @@ -133,7 +133,7 @@ func TestSupplierPoolConversionMongoStandalone(t *testing.T) { if _, err := restarted.ConvertSupplierPool(ctx, org, project, secondToken.PreviewToken, actor); err != ErrSupplierPoolConversionPending { t.Fatalf("second token started parallel conversion: %v", err) } - for _, col := range []*mongo.Collection{s.humanBatches, s.annotationTasks} { + for _, col := range []mongoCollection{s.humanBatches, s.annotationTasks} { id := "batch-interrupted" if col == s.annotationTasks { id = "task-interrupted" diff --git a/internal/store/workforce_stats.go b/internal/store/workforce_stats.go index 0ca3687f..a723d16b 100644 --- a/internal/store/workforce_stats.go +++ b/internal/store/workforce_stats.go @@ -11,7 +11,6 @@ import ( "droboticflow/internal/domain" "droboticflow/internal/workforcestats" "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) @@ -259,6 +258,11 @@ type humanWorkAggregateRow struct { // MongoStore implementation pushes the counting into aggregation pipelines so // the dashboard never materializes humanItems/deliveryPackages documents. func (s *MongoStore) DashboardHumanWorkStats(ctx context.Context, organizationID string) (DashboardHumanWorkStats, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return DashboardHumanWorkStats{}, err + } else if replica != nil { + return replica.DashboardHumanWorkStats(ctx, organizationID) + } stats := DashboardHumanWorkStats{} baseFilter := humanItemMongoFilter(organizationID, domain.HumanItemQuery{}) rows := []humanWorkAggregateRow{} @@ -392,6 +396,11 @@ var workforceStatsCache = &ttlResultCache[workforcestats.Snapshot]{now: time.Now // miss. MemoryStore intentionally has no cached variant: in-process tests and // the local replica expect read-after-write freshness. func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return workforcestats.Snapshot{}, err + } else if replica != nil { + return s.loadReplicaWorkforce(ctx, replica, org, q) + } allowed := make([]string, 0, len(q.AllowedProjects)) for id, on := range q.AllowedProjects { if on { @@ -421,6 +430,11 @@ func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q // Read-only, time-bounded event queries and batched joins. No index migrations or // customer data rewrites. History queries project out media and annotation text. func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) { + if replica, err := s.reportReplica(ctx); err != nil { + return workforcestats.Snapshot{}, err + } else if replica != nil { + return s.loadReplicaWorkforce(ctx, replica, org, q) + } d := workforcestats.Snapshot{} fields := func(names ...string) bson.M { m := bson.M{} @@ -429,7 +443,7 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf } return m } - read := func(c *mongo.Collection, filter, projection bson.M, target any) error { + read := func(c mongoCollection, filter, projection bson.M, target any) error { cur, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second)) if err != nil { return err @@ -493,7 +507,7 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf projection["effects.review.comment"] = 1 projection["effects.reject.comment"] = 1 } - if err := read(s.db.Collection("reviewOperations"), organizationScopedFilter(org, of), projection, &d.Operations); err != nil { + if err := read(s.queryCollection("reviewOperations"), organizationScopedFilter(org, of), projection, &d.Operations); err != nil { return d, err } af := window("completedAt") @@ -509,7 +523,7 @@ func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workf af["status"] = "completed" af["decision"] = bson.M{"$in": bson.A{"approved", "rejected"}} af["stageId"] = bson.M{"$in": bson.A{"acceptance", "prepackage_acceptance"}} - if err := read(s.db.Collection("reviewOperations"), organizationScopedFilter(org, af), fields("_id", "organizationId", "projectId", "userId", "stageId", "status", "decision", "createdAt", "completedAt", "sourceItemIds", "reviewVideoSources"), &d.AcceptanceDecisions); err != nil { + if err := read(s.queryCollection("reviewOperations"), organizationScopedFilter(org, af), fields("_id", "organizationId", "projectId", "userId", "stageId", "status", "decision", "createdAt", "completedAt", "sourceItemIds", "reviewVideoSources"), &d.AcceptanceDecisions); err != nil { return d, err } if sourceIDs := workforceAcceptanceSourceIDs(d.AcceptanceDecisions); len(sourceIDs) > 0 { diff --git a/scripts/local/query-replica.compose.yml b/scripts/local/query-replica.compose.yml new file mode 100644 index 00000000..eb5c6481 --- /dev/null +++ b/scripts/local/query-replica.compose.yml @@ -0,0 +1,31 @@ +name: fanyu-query-replica-local +services: + valkey: + image: valkey/valkey@sha256:e51a82741b780e4bf315db10753edf07eea6496d405fa7df2a8677a18f5e7464 + command: + ["valkey-server", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru", "--save", "", "--appendonly", "no"] + ports: ["127.0.0.1:27139:6379"] + cpus: 1 + mem_limit: 256m + source: + image: mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b + command: ["mongod", "--port", "27119", "--bind_ip_all", "--wiredTigerCacheSizeGB", "1"] + ports: ["127.0.0.1:27119:27119"] + cpus: 4 + mem_limit: 4g + volumes: ["source-data:/data/db"] + ulimits: + nofile: { soft: 65536, hard: 65536 } + query: + image: mongo@sha256:5d7043a4ffe02b9ed1b6e0bab057546981af5ca0a79107e9c461e49bc44c0a7b + command: + ["mongod", "--port", "27129", "--replSet", "query-read-rs", "--bind_ip_all", "--wiredTigerCacheSizeGB", "2"] + ports: ["127.0.0.1:27129:27129"] + cpus: 4 + mem_limit: 8g + volumes: ["query-data:/data/db"] + ulimits: + nofile: { soft: 65536, hard: 65536 } +volumes: + source-data: + query-data: diff --git a/scripts/test_query_replica.py b/scripts/test_query_replica.py new file mode 100644 index 00000000..9961c01b --- /dev/null +++ b/scripts/test_query_replica.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Owned loopback-only replica lab; no company credentials or customer databases.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import re +import signal +import socket +import subprocess +import time +from lib.release_checks import check_environment, source_fingerprint, write_log + +ROOT = Path(__file__).resolve().parents[1] +PRIVATE = ROOT / 'local-data/query-replica-local' +COMPOSE = ROOT / 'scripts/local/query-replica.compose.yml' +CASES = ('internal/store/query_replica_local_test.go', + 'internal/store/query_sync_local_test.go', + 'internal/store/query_replica_count_cache_test.go', + 'internal/api/query_replica_local_test.go', + 'cmd/query-cdc-probe/main_test.go', + 'internal/store/query_replica_valkey_local_test.go') + + +def owned_native(): + for folder, port in [('source', 27119), ('read', 27129)]: + try: + pid = (PRIVATE / folder / 'mongod.pid').read_text().strip() + if not pid.isdecimal(): + return False + command = subprocess.check_output(['ps', '-p', pid, '-o', 'command='], text=True, stderr=subprocess.DEVNULL) + if str(PRIVATE / folder) not in command or f'--port {port}' not in command: + return False + except (OSError, subprocess.SubprocessError): + return False + return True + + +def run(argv, env, timeout=660): + child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = child.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGTERM) + try: + stdout, stderr = child.communicate(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGKILL) + stdout, stderr = child.communicate() + return 2, stdout + '\n' + stderr + '\ncheck timed out' + return child.returncode, stdout + '\n' + stderr + + +def compose(env, *args): + # Refuse remote Docker contexts before performing any container mutation. + endpoint = subprocess.check_output(['docker', 'context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'], + cwd=ROOT, env=env, stderr=subprocess.DEVNULL, text=True).strip() + if not endpoint.startswith(('unix://', 'npipe://')): + raise RuntimeError('only a local Docker socket is supported') + code, output = run(['docker', 'compose', '-f', str(COMPOSE), *args], env, 180) + if code: + proof = write_log(PRIVATE / 'logs', ['docker', 'compose', *args], output, 'failed') + raise RuntimeError('local dependency command failed; ' + proof['log']) + return output + + +def start(env): + # Existing unowned listeners must not be replaced or stopped. + busy = [] + for port in (27119, 27129): + try: + with socket.create_connection(('127.0.0.1', port), timeout=.3): + busy.append(port) + except OSError: + pass + if busy: + running = compose(env, 'ps', '--status', 'running', '--services').splitlines() + if not {'source', 'query'} <= set(running): + raise RuntimeError('lab ports are occupied by unowned services; use an owned native lab or free these ports') + compose(env, 'up', '-d') + for _ in range(60): + try: + result = compose(env, 'exec', '-T', 'query', 'mongosh', '--quiet', '--port', '27129', '--eval', + "try {rs.status()} catch(e) {rs.initiate({_id:'query-read-rs',members:[{_id:0,host:'127.0.0.1:27129'}]})}; if (!db.hello().isWritablePrimary) quit(2)") + if result: + return + except RuntimeError: + time.sleep(1) + raise RuntimeError('query replica did not become primary') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--native', action='store_true') + parser.add_argument('--start', action='store_true') + parser.add_argument('--auto', action='store_true') + parser.add_argument('--down', action='store_true') + parser.add_argument('--race', action='store_true') + parser.add_argument('--scale', action='store_true') + parser.add_argument('--cache', action='store_true') + args = parser.parse_args() + PRIVATE.mkdir(parents=True, exist_ok=True, mode=0o700) + env = {k: v for k, v in check_environment().items() if not k.startswith('QUERY_')} + env['QUERY_REPLICA_LOCAL_TEST'] = '1' + before = source_fingerprint(ROOT) + with (PRIVATE / 'runner.lock').open('w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if args.down: + compose(env, 'down') + return 0 + native = args.native or (args.auto and owned_native()) + if native: + if not owned_native(): + raise RuntimeError('native mode requires this worktree\'s two owned Mongo processes') + elif args.start or args.auto: + start(env) + env['QUERY_REPLICA_DOCKER_LAB'] = '1' + else: + raise RuntimeError('select --native, --start or --auto explicitly') + if args.cache: + compose(env, 'up', '-d', 'valkey') + env['QUERY_REPLICA_VALKEY_LOCAL_TEST'] = '1' + required = set() + for name in CASES: + required.update(re.findall(r'^func (Test\w+)\(', (ROOT / name).read_text(), re.M)) + if not args.cache: + required = {name for name in required if not name.startswith('TestQueryReplicaSharedValkey')} + if args.scale: + required = {'TestQueryReplicaScaleLocal'} + env['QUERY_REPLICA_SCALE_LOCAL'] = '1' + regex = '^(' + '|'.join(sorted(required)) + ')$' + argv = ['go', 'test', './internal/store', './internal/api', './cmd/query-cdc-probe', + '-run', regex, '-count=1', '-timeout=10m', '-v'] + if args.race: + argv.append('-race') + code, output = run(argv, env) + passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M)) + missing = sorted(required - passed) + changed = not before or source_fingerprint(ROOT) != before + skipped = bool(re.search(r'^--- SKIP:', output, re.M)) + success = code == 0 and not missing and not changed and not skipped + proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed') + proof.update(missing=missing, sourceChanged=changed, skipped=skipped, + environment='native-uncapped' if native else 'docker-source-4cpu-query-4cpu', + productionCapacityAccepted=False) + print(json.dumps(proof, ensure_ascii=False)) + return 0 if success else (code or 2) + + +if __name__ == '__main__': + try: + raise SystemExit(main()) + except (RuntimeError, OSError, subprocess.SubprocessError) as error: + print(str(error)) + raise SystemExit(2) diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 1f3c19dc..21d35873 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -900,28 +900,14 @@ ], "frontend": [], "go": { - "./internal/queryserving": [ - "^TestQueryServing" - ], - "./internal/store": [ - "^TestQueryReadModel", - "^TestQuerySearch" - ], - "./internal/outputsnapshot": [ - "^TestQueryServing" - ], - "./internal/api": [ - "^TestQueryServing" - ] + "./internal/queryserving": ["^TestQueryServing"], + "./internal/store": ["^TestQueryReadModel", "^TestQuerySearch"], + "./internal/outputsnapshot": ["^TestQueryServing"], + "./internal/api": ["^TestQueryServing"] }, "task": [ { - "argv": [ - "python3", - "scripts/test_query_serving.py", - "--start", - "--race" - ], + "argv": ["python3", "scripts/test_query_serving.py", "--start", "--race"], "tests": true, "env": {} }, @@ -939,8 +925,36 @@ "tests": true } ], + "warnings": ["真实三组件仅使用固定本机合成夹具;完整规模三轮、故障恢复容量及生产部署独立验收。"] + }, + { + "name": "independent-query-replica", + "paths": [ + "^internal/store/query_(replica|sync|tracked_collection|collection_interface).*\\.go$", + "^internal/api/query_replica.*\\.go$", + "^cmd/(query-replica|query-cdc-probe)/", + "^scripts/test_query_replica\\.py$", + "^scripts/local/query-replica\\.compose\\.yml$", + "^Dockerfile\\.query-replica$", + "^docs/(adr/0014-.*|operations/query-replica\\.md)$" + ], + "frontend": [], + "go": { + "./internal/store": [ + "^TestQueryReplicaCandidateCache", + "^TestQueryReplicaMissingConfiguration", + "^TestQueryReplicaRejectsSigned" + ], + "./cmd/query-cdc-probe": ["^TestClassification"] + }, + "task": [ + { + "argv": ["python3", "scripts/test_query_replica.py", "--auto", "--race", "--cache"], + "tests": true + } + ], "warnings": [ - "真实三组件仅使用固定本机合成夹具;完整规模三轮、故障恢复容量及生产部署独立验收。" + "独立源库/查询库仅使用合成数据;生产写者、47个源索引、迁移及集群资源须另行审批。native测试没有CPU配额,不能证明4核容量。" ] } ]