diff --git a/.scratch/query-recovery-hardening/work.md b/.scratch/query-recovery-hardening/work.md new file mode 100644 index 00000000..5667c3d2 --- /dev/null +++ b/.scratch/query-recovery-hardening/work.md @@ -0,0 +1,30 @@ +# Query recovery, role checks and cutover safety + +Status: local implementation and scoped verification complete; not deployed. + +Base: dev d0dfddaf; unified recent-integration. User authorized fixes and validation, then release preparation. No production query switch, customer-data writes, index build or cluster changes were performed. Prior count/cache/page-scan and UI improvements remain. Protected pre-existing AGENTS/docs/admin-pool changes are excluded. + +## Changes + +- SnapshotTooOld (Mongo239), before inputs are sealed: persist the original failure, yield, then replace only the unfinished capture generation with a fresh query-DB snapshot. Preserve source synchronization checkpoints, confirmed facts, published heads and retained input pages until replacement completes. Do not combine pages from different snapshots or re-bootstrap original Mongo. Three replacements without sealing remain a genuine no-progress failure. Sealed plans retain their existing recovery path. +- Transient failure reset requires the model lease and majority+journal transaction; it cannot manufacture progress or clear lease/persistence/no-progress failures. Projector read retries remain; hard failures stop visibly. +- Role changes use authoritative scoped existence queries with ID-only projection instead of loading an entire organization's payloads. The transaction rechecks the same rule against competing claims. Submitted history, unfinished-work blocking, legacy statuses and organization isolation are preserved. Proposed optional covering indexes are not startup migrations and require their own production plan. +- Shadow startup retains the observer, safety run, approved cumulative budget, nonce and finite window. Mutations require an approved immutable plan, current synchronized source and UID/RV checks. Metadata probes get bounded diagnosed retries. Production cutover additionally requires actual live result and permission evidence; ready Pods/metadata/synthetic results do not qualify. Approved100/150/200 throughput/batch stages are preserved; there is no extra blanket100-row cap. + +## Evidence + +Evidence folder: local-data/test-results/query-recovery-hardening. + +- final-task/report.json: other selected checks passed, including Go known-failure gate, API permissions/history/role guards, concurrent task-pool read mix, query-serving/cache/authorization and observer/release-helper checks. Two initial failures are retained: an incorrect explicit-null legacy fixture and Docker mount-array order comparison. No business rule was changed to satisfy those fixtures. +- affected-rerun/report.json: both corrected cases and the complete registered isolated suite passed.116/116, missing=[], skipped=false, sourceChanged=false;6 bounded batches and6 same-volume/resource majority+journal canary recoveries. Batching does not establish continuous capacity. +- cutover-final.log:19 Python protection cases passed, including approved100/150/200 stages, denied/expired scope, result/permission mismatch, mixed cohorts, probe retries and CAS behavior. Last changes only affect this Python helper/test; Go inputs are unchanged from the116-case run. +- affected-rerun/02.log:12,000 synthetic tasks; old227ms/14,293,087 reply bytes versus new4ms/515 bytes. Before adding this candidate index, new query took2ms. Legacy missing/empty/default organization and status checks passed. Not a production latency claim. +- same-scale.log:35,000 events,140,000 facts; reused125,186 and wrote14,814 across2 rounds in77.50s. Refreshes170.67ms/162.14ms; original-Mongo reads0, fixed512KiB/s and majority+journal unchanged. This is local1CPU/1Gi, not production offload acceptance. +- physical-restart.log: actual owned-container restart mid-apply, same volume/resources,408 facts recovered, repeated native input reads0, majority+journal preserved. +- compile.log: all cmd/internal packages compiled. Source/test/environment identities and original failed runs remain in the evidence folder. Final source digest is recorded separately. + +## Release preparation and remaining scope + +Proposed sequence is recorded in local-data/test-results/query-recovery-hardening/release-preparation.md. Release role-check fix with existing production read flags and advisory configuration unchanged; update/resume shadow components in a separate approved plan/window. Preserve all checkpoints and published results. Production result/permission comparison and measured original-Mongo offload remain pending; no claim of completed cutover or capacity acceptance. + +Need explicit authorization for candidate push, clean release worktree and image uploads; then exact synchronized dev + immutable digests, formal release gates and fresh target/in-flight checks before presenting final deployment plans. Proposed account indexes require separate explain/build authorization; they are not automatically created. No existing customer annotations, ownership, review records or production data were reset or overwritten. diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go index 03dc4018..8a3801d8 100644 --- a/cmd/query-replica/main.go +++ b/cmd/query-replica/main.go @@ -297,6 +297,11 @@ func main() { d := store.QueryReplicaDiagnostic(e) log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode) if *safetyRun == "" && !*once { + if !daemonRetryable(e) { + log.Print("projector stopped on hard failure; checkpoint retained") + <-ctx.Done() + return + } wait := daemonRetryDelay(consecutiveFailures) consecutiveFailures++ timer := time.NewTimer(wait) @@ -344,18 +349,9 @@ func main() { if e := q.WaitNextPass(workCtx, delay); e != nil { if ctx.Err() == nil { if *safetyRun == "" && !*once { - log.Print("projector lease refresh retry") - wait := daemonRetryDelay(consecutiveFailures) - consecutiveFailures++ - timer := time.NewTimer(wait) - select { - case <-workCtx.Done(): - timer.Stop() - return - case <-timer.C: - } - _ = q.Acquire(workCtx) - continue + log.Print("projector stopped on lease/persistence failure; checkpoint retained") + <-ctx.Done() + return } pause("target_error") log.Print("projector lease unavailable; checkpoint retained") diff --git a/cmd/query-replica/pacing.go b/cmd/query-replica/pacing.go index 16779bea..73bd68d0 100644 --- a/cmd/query-replica/pacing.go +++ b/cmd/query-replica/pacing.go @@ -1,6 +1,9 @@ package main -import "time" +import ( + "droboticflow/internal/store" + "time" +) // The default preserves immediate periodic backlog draining. A production // backfill can explicitly pace every pass without changing query timeouts, @@ -44,3 +47,13 @@ func adaptiveBackfillCooldown(ready bool, elapsed time.Duration) time.Duration { } return 0 } + +// Retrying transport reads is safe; permission, lease, unsupported data and +// persistence failures must remain visible to the operator/health observer. +func daemonRetryable(err error) bool { + d := store.QueryReplicaDiagnostic(err) + if d.Phase == "target_apply" || d.Phase == "checkpoint" || d.Class == "mongo_write" || d.Class == "mongo_bulk_write" { + return false + } + return d.Class == "network" || d.Class == "timeout" || d.Class == "deadline" +} diff --git a/cmd/query-replica/pacing_test.go b/cmd/query-replica/pacing_test.go index edfd9456..ab0bdc84 100644 --- a/cmd/query-replica/pacing_test.go +++ b/cmd/query-replica/pacing_test.go @@ -1,6 +1,9 @@ package main import ( + "context" + "droboticflow/internal/store" + "go.mongodb.org/mongo-driver/mongo" "testing" "time" ) @@ -72,3 +75,14 @@ func TestDaemonRetryDelayAndAdaptiveBackfillCooldown(t *testing.T) { } } } + +func TestDaemonRetryDoesNotHideHardFailures(t *testing.T) { + for _, e := range []error{store.ErrQueryReplicaLease, store.ErrQueryReplicaUnsupported, mongo.CommandError{Code: 13}, mongo.WriteException{WriteConcernError: &mongo.WriteConcernError{Code: 64}}} { + if daemonRetryable(e) { + t.Fatal("hard failure retried", e) + } + } + if !daemonRetryable(context.DeadlineExceeded) { + t.Fatal("read timeout stopped daemon") + } +} diff --git a/docs/adr/0014-independent-query-database.md b/docs/adr/0014-independent-query-database.md index b8b24b27..fbf3c697 100644 --- a/docs/adr/0014-independent-query-database.md +++ b/docs/adr/0014-independent-query-database.md @@ -81,4 +81,4 @@ 输入捕获和计划保存在私有`queryBuildJobs`/`queryBuildParts`。原生输入分段、游标以及写入批次、应用游标分别在同一`majority+journal`事务确认。续建重新核对封存计划指纹和e4暂存事实哈希;未发布的大块结果由清理程序保留。派生事实采用原有版本可见性规则,全部完成及行数核对后才在同一事务发布两个统计头。已追平观察器证明业务输入未变化时,只重新确认完整结果,不重新读取历史、计算或遍历全量事实。 -输入快照的逻辑时间与展示时间分开保存。已经封存输入后,恢复不依赖原快照仍在Mongo历史窗口内;尚未捕获完的输入若遇到快照历史丢失,必须保留分段并明确停止,不能切换时间点后拼接新旧结果。该异常不自动重新回填权威库。首次上线前须检查查询库快照/oplog保留窗口,不能把任意长停机期间未封存输入的自动恢复视为已经覆盖。 +输入快照的逻辑时间与展示时间分开保存。已经封存输入后,恢复不依赖原快照仍在Mongo历史窗口内;尚未捕获完的输入遇到快照历史丢失时,不得切换时间点后拼接新旧结果。2026-11-16用户批准的恢复规则:保存Mongo239失败与原分段,在新快照下新建本组织派生统计代次,保留旧完整结果并按内容指纹复用已确认事实;不重新回填权威库。连续三次未能封存仍明确停止。已封存输入继续从持久游标恢复。详见`docs/operations/query-build-lifecycle.md`;实现和本地验证不代表生产启用。 diff --git a/docs/operations/account-role-pending-indexes.json b/docs/operations/account-role-pending-indexes.json new file mode 100644 index 00000000..ea241a9e --- /dev/null +++ b/docs/operations/account-role-pending-indexes.json @@ -0,0 +1,8 @@ +{ + "purpose": "Authoritative role-change existence checks; prepare only, no automatic startup migration", + "database": "drobodata-fanyu", + "collections": ["humanItems", "annotationTasks"], + "index": {"name": "account_role_pending_v1", "key": {"organizationId": 1, "assignedUserId": 1, "status": 1, "_id": 1}}, + "productionChanges": "Require separately approved plan, per-collection build and read-only explain comparison; not executed by this change", + "rollback": "Drop only these newly created indexes if required; no data rewrite, no hint requirement" +} diff --git a/docs/operations/query-build-lifecycle.md b/docs/operations/query-build-lifecycle.md index 5afcedaf..7b066c29 100644 --- a/docs/operations/query-build-lifecycle.md +++ b/docs/operations/query-build-lifecycle.md @@ -80,3 +80,17 @@ 不再以首次构建5分钟、1Gi实验库连续全套运行、全部查询500ms或200/500人长时压测阻塞影子部署。先更新影子构建器,在有效新窗口中追平同步,确认一个完整发布及关键生产对账,再灰度接入少量看板/作业记录。保留现有3分钟后台刷新、15分钟最大展示年龄;首轮以首屏约2–3秒、无超时、结果和权限正确为目标。全量入口和长期容量须后续独立验收,不能把分批功能通过换算成容量承诺。 不可放宽项:majority+journal、权限/供应商隔离、版本竞争、客户标注保护、完整代次原子发布、硬故障停止与回退。客户现有标注、切片、送检、质检和验收继续使用现有生产链路,不等待新查询链路完成。原运行窗口已结束,新的启动截止、镜像及目标仍须按项目规则确认。 + +## 2026-11-16:未封存快照恢复与发布保护修复(本地,尚未启用) + +`SnapshotTooOld`(Mongo 239)仅在 `production_plan` / `workforce_plan` 的未封存输入阶段自动恢复:先以原持久性记录失败,正常让出;下轮在同一构建租约保护下选择新快照、新输入/计划代次,只重建该组织未完成的派生统计。原同步事实、源同步断点、累计预算、已发布头、事实内容及权限不变。前一代捕获页和未发布生产块在替代代次完成前保留,之后由原有有界清理回收;故障来源以 `snapshotRecoveries` 保存。 + +不能把不同快照的捕获页混用,因此未封存部分可能重新读取**查询库中的当前组织输入**;这不等于重新扫描/回填原Mongo。已经封存的计划照旧从应用游标续建,指纹一致的事实不重写。连续三次仍无法封存输入才报告无进展;不增加读取/事务超时、不调整512KiB/s或majority+journal。 + +网络/读取超时保留冷却重试。租约、权限、数据损坏、写入/持久化失败和真实无进展不自动清掉;清理临时故障标记也必须取得当前构建租约并使用majority+j事务,不重置进展时间或累计无进展值。普通单轮预算到期仍只让出。 + +`evening-query-cutover.py` 的写操作须提供已审查JSON计划、完整SHA256确认及授权消息引用。计划绑定集群context/API server摘要、namespace UID、对象UID、不可变镜像、有限累计预算/速率及明确有效期。JSON Patch通过stdin发送,核对原读取的resourceVersion;暂停/缩放同样用UID/RV CAS。不再停用观察器来强行启动;沿用有效的维护期CPU观察策略及其他硬故障保护。未获生产切换授权时不得启用生产入口。 + +`verify-parity` 不再把Ready/非空当结果一致:实际切换还须携带绑定当前已发布头与事实代次的新旧结果摘要、组织/供应商权限范围摘要及授权/拒绝结果,覆盖五角色、计数/看板/作业记录、正常/旧数据/日期边界/删除/返工/撤权/跨供应商/跨组织。合成证据仅用于本地验证,不能充当线上对照。后台首次准备和结果就绪只表示各自阶段,不代表已经分流或源库降载。 + +用户本轮仅授权本地修复和隔离验证,**不启动生产同步,不建生产索引,不切查询入口**。新维护窗口、镜像和具体回退仍在完整发布门禁后列入最终计划确认。 diff --git a/internal/api/account_hierarchy.go b/internal/api/account_hierarchy.go index 1aa42228..0c6fcec1 100644 --- a/internal/api/account_hierarchy.go +++ b/internal/api/account_hierarchy.go @@ -13,28 +13,15 @@ func (s *server) validateAccountSupplier(ctx context.Context, org, id string, ro return s.validateUserSupplier(ctx, org, id) } func (s *server) ensureRoleChangeIdle(w http.ResponseWriter, r *http.Request, user domain.UserRecord) bool { - items, err := s.store.ListHumanItems(r.Context(), user.OrganizationID, domain.HumanItemQuery{}) + pending, err := s.store.AccountHasPendingWork(r.Context(), user.OrganizationID, user.ID, user.Role) if err != nil { - renderError(w, 500, err) + renderError(w, http.StatusInternalServerError, err) return false } - for _, item := range items { - if item.AssignedUserID == user.ID && humanItemPending(item.Status) { - renderJSON(w, 409, map[string]string{"code": "account_has_pending_tasks", "error": "该账号有未完成的已领取任务,请先提交或释放任务后再修改角色"}) - return false - } - } - tasks, err := s.store.ListAnnotationTasks(r.Context(), user.OrganizationID, "") - if err != nil { - renderError(w, 500, err) + if pending { + renderJSON(w, http.StatusConflict, map[string]string{"code": "account_has_pending_tasks", "error": "该账号有未完成任务,请先提交或释放任务后再修改角色"}) return false } - for _, task := range tasks { - if task.AssignedUserID == user.ID && domain.AnnotationTaskBlocksRoleChange(task.Status, user.Role) { - renderJSON(w, 409, map[string]string{"code": "account_has_pending_tasks", "error": "该账号有关联标注任务尚未完成,请先处理任务"}) - return false - } - } return true } diff --git a/internal/api/account_role_guard_bounded_test.go b/internal/api/account_role_guard_bounded_test.go new file mode 100644 index 00000000..f6df2ad4 --- /dev/null +++ b/internal/api/account_role_guard_bounded_test.go @@ -0,0 +1,32 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "droboticflow/internal/store" + "net/http/httptest" + "testing" +) + +type roleGuardDiagnosticStore struct { + store.Store + unscoped bool +} + +func (s *roleGuardDiagnosticStore) ListHumanItems(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) { + if q.ActorUserID == "" { + s.unscoped = true + return nil, context.DeadlineExceeded + } + return s.Store.ListHumanItems(ctx, org, q) +} +func TestRoleChangeIdleGuardAvoidsOrganizationWideRead(t *testing.T) { + p := &roleGuardDiagnosticStore{Store: store.NewMemoryStore()} + s := &server{store: p} + w := httptest.NewRecorder() + r := httptest.NewRequest("PUT", "/api/users/diagnostic-quality", nil) + user := domain.UserRecord{ID: "diagnostic-quality", OrganizationID: "diagnostic-org", Role: domain.UserRoleQualityReviewer} + if !s.ensureRoleChangeIdle(w, r, user) { + t.Fatalf("idle role change failed: status=%d orgWideRead=%v; guard loaded all organization tasks before checking current user", w.Code, p.unscoped) + } +} diff --git a/internal/store/account_role_change.go b/internal/store/account_role_change.go index cce48bf0..a7b28453 100644 --- a/internal/store/account_role_change.go +++ b/internal/store/account_role_change.go @@ -5,6 +5,8 @@ import ( "droboticflow/internal/domain" "errors" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "reflect" ) @@ -27,18 +29,11 @@ func (s *MongoStore) updateAccountRole(ctx context.Context, item, expected domai if user == nil || !sameUserMutationSnapshot(*user, expected) { return nil, ErrUserMutationConflict } - n, e := s.humanItems.CountDocuments(sc, bson.M{"assignedUserId": item.ID, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemClaimed, domain.HumanItemInProgress, domain.HumanItemSubmitted, domain.HumanItemRework, domain.HumanItemEscalated}}}) + pending, e := s.AccountHasPendingWork(sc, item.OrganizationID, item.ID, expected.Role) if e != nil { return nil, e } - if n > 0 { - return nil, ErrAccountPendingTasks - } - n, e = s.annotationTasks.CountDocuments(sc, bson.M{"assignedUserId": item.ID, "status": bson.M{"$nin": domain.AnnotationRoleChangeNonBlockingStatuses(expected.Role)}}) - if e != nil { - return nil, e - } - if n > 0 { + if pending { return nil, ErrAccountPendingTasks } return nil, s.upsertUser(sc, item, &expected) @@ -240,3 +235,46 @@ func (s *MongoStore) updateAnnotationClaimVersioned(ctx context.Context, item do } return updated, err } + +// This is an authoritative existence check, never a cached eligibility decision. +func (s *MongoStore) AccountHasPendingWork(ctx context.Context, org, user string, role domain.UserRole) (bool, error) { + filters := []struct { + collection mongoCollection + filter bson.M + }{ + {s.humanItems, organizationScopedFilter(org, bson.M{"assignedUserId": user, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemClaimed, domain.HumanItemInProgress, domain.HumanItemSubmitted, domain.HumanItemRework, domain.HumanItemEscalated}}})}, + {s.annotationTasks, organizationScopedFilter(org, bson.M{"assignedUserId": user, "status": bson.M{"$nin": domain.AnnotationRoleChangeNonBlockingStatuses(role)}})}, + } + for _, q := range filters { + var row bson.M + e := q.collection.FindOne(ctx, q.filter, options.FindOne().SetProjection(bson.M{"_id": 1})).Decode(&row) + if e == nil { + return true, nil + } + if !errors.Is(e, mongo.ErrNoDocuments) { + return false, e + } + } + return false, nil +} +func (s *MemoryStore) accountHasPendingWorkLocked(org, user string, role domain.UserRole) bool { + for _, v := range s.humanItems { + if humanOrganizationMatches(v.OrganizationID, org) && v.AssignedUserID == user && pendingAccountItem(v.Status) { + return true + } + } + for _, v := range s.annotationTasks { + if humanOrganizationMatches(v.OrganizationID, org) && v.AssignedUserID == user && domain.AnnotationTaskBlocksRoleChange(v.Status, role) { + return true + } + } + return false +} +func (s *MemoryStore) AccountHasPendingWork(ctx context.Context, org, user string, role domain.UserRole) (bool, error) { + if e := ctx.Err(); e != nil { + return false, e + } + s.mu.Lock() + defer s.mu.Unlock() + return s.accountHasPendingWorkLocked(org, user, role), nil +} diff --git a/internal/store/account_role_pending_local_test.go b/internal/store/account_role_pending_local_test.go new file mode 100644 index 00000000..ee60b0fc --- /dev/null +++ b/internal/store/account_role_pending_local_test.go @@ -0,0 +1,102 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestAccountRolePendingBoundedParityLocal(t *testing.T) { + var replyBytes atomic.Int64 + monitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { + if e.CommandName == "find" || e.CommandName == "getMore" { + replyBytes.Add(int64(len(e.Reply))) + } + }} + _, s, _ := newLocalReplica(t, nil, monitor) + ctx := t.Context() + org := "role-org" + index := mongo.IndexModel{Keys: bson.D{{Key: "organizationId", Value: 1}, {Key: "assignedUserId", Value: 1}, {Key: "status", Value: 1}, {Key: "_id", Value: 1}}, Options: options.Index().SetName("account_role_pending_v1")} + + docs := make([]any, 12000) + for i := range docs { + docs[i] = bson.M{"id": fmt.Sprintf("other-%06d", i), "organizationId": org, "assignedUserId": "other", "status": "accepted", "subject": bson.M{"payload": bson.M{"padding": strings.Repeat("x", 1024)}}} + } + if _, e := s.humanItems.InsertMany(ctx, docs); e != nil { + t.Fatal(e) + } + replyBytes.Store(0) + start := time.Now() + all, e := s.ListHumanItems(ctx, org, domain.HumanItemQuery{}) + oldMS := time.Since(start).Milliseconds() + if e != nil || len(all) != 12000 { + t.Fatal(e) + } + oldBytes := replyBytes.Swap(0) + start = time.Now() + pending, e := s.AccountHasPendingWork(ctx, org, "worker", domain.UserRoleQualityReviewer) + unindexedMS := time.Since(start).Milliseconds() + if e != nil || pending { + t.Fatal(pending, e) + } + replyBytes.Store(0) + for _, c := range []mongoCollection{s.humanItems, s.annotationTasks} { + if _, e := c.Indexes().CreateOne(ctx, index); e != nil { + t.Fatal(e) + } + } + start = time.Now() + pending, e = s.AccountHasPendingWork(ctx, org, "worker", domain.UserRoleQualityReviewer) + newMS := time.Since(start).Milliseconds() + if e != nil || pending { + t.Fatal(pending, e) + } + newBytes := replyBytes.Load() + if newBytes*100 >= oldBytes { + t.Fatal("existence check still transfers large payload", oldBytes, newBytes) + } + for _, tc := range []struct { + status domain.AnnotationTaskStatus + role domain.UserRole + want bool + }{{domain.AnnotationTaskStatusPendingQuality, domain.UserRoleAnnotator, false}, {domain.AnnotationTaskStatusPendingAcceptance, domain.UserRoleAnnotator, false}, {domain.AnnotationTaskStatusPendingQuality, domain.UserRoleQualityReviewer, true}, {"unknown-legacy", domain.UserRoleAnnotator, true}, {domain.AnnotationTaskStatusAccepted, domain.UserRoleQualityReviewer, false}} { + _, e = s.annotationTasks.ReplaceOne(ctx, bson.M{"id": "legacy"}, bson.M{"id": "legacy", "organizationId": org, "assignedUserId": "worker", "status": tc.status}, options.Replace().SetUpsert(true)) + if e != nil { + t.Fatal(e) + } + got, e := s.AccountHasPendingWork(ctx, org, "worker", tc.role) + if e != nil || got != tc.want { + t.Fatal(tc, got, e) + } + if got, e = s.AccountHasPendingWork(ctx, "other-org", "worker", tc.role); e != nil || got { + t.Fatal("cross organization work leaked", got, e) + } + } + for _, legacyOrg := range []any{nil, "", domain.DefaultOrganizationID} { + doc := bson.M{"id": "legacy-org", "assignedUserId": "legacy-worker", "status": domain.HumanItemClaimed} + if legacyOrg != nil { + doc["organizationId"] = legacyOrg + } + if _, e = s.humanItems.ReplaceOne(ctx, bson.M{"id": "legacy-org"}, doc, options.Replace().SetUpsert(true)); e != nil { + t.Fatal(e) + } + got, e := s.AccountHasPendingWork(ctx, domain.DefaultOrganizationID, "legacy-worker", domain.UserRoleAnnotator) + if e != nil || !got { + t.Fatal("legacy organization state lost", got, e) + } + } + cancelled, stop := context.WithCancel(ctx) + stop() + if _, e = s.AccountHasPendingWork(cancelled, org, "worker", domain.UserRoleAnnotator); e == nil { + t.Fatal("cancel ignored") + } + t.Logf("same 12000 rows: old_ms=%d new_ms=%d old_read_bytes=%d new_read_bytes=%d unindexed_new_ms=%d; status/legacy/organization parity passed", oldMS, newMS, oldBytes, newBytes, unindexedMS) +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 5d8eb0b4..d15a7fdf 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -2397,15 +2397,8 @@ func (store *MemoryStore) upsertUser(_ context.Context, item domain.UserRecord, return ErrAcceptanceAuthorizationRequired } item.AcceptanceAccessMode = "managed" - for _, task := range store.humanItems { - if task.AssignedUserID == item.ID && pendingAccountItem(task.Status) { - return ErrAccountPendingTasks - } - } - for _, task := range store.annotationTasks { - if task.AssignedUserID == item.ID && domain.AnnotationTaskBlocksRoleChange(task.Status, expected.Role) { - return ErrAccountPendingTasks - } + if store.accountHasPendingWorkLocked(item.OrganizationID, item.ID, expected.Role) { + return ErrAccountPendingTasks } } if existing, ok := store.users[item.ID]; ok && removesActiveAdmin(existing, item) { diff --git a/internal/store/query_build_jobs.go b/internal/store/query_build_jobs.go index 32560e30..ca751fe2 100644 --- a/internal/store/query_build_jobs.go +++ b/internal/store/query_build_jobs.go @@ -23,7 +23,18 @@ type queryBuildRetiredGeneration struct { Until time.Time `bson:"until"` } +// Retained until a replacement cohort commits; never combines old and new input pages. +type queryBuildSnapshotRecovery struct { + Code int32 `bson:"code"` + RecoveredAt time.Time `bson:"recoveredAt"` + Generation string `bson:"generation"` + Snapshot primitive.Timestamp `bson:"snapshot"` + AsOf time.Time `bson:"asOf"` + Phase string `bson:"phase"` +} + type queryBuildJob struct { + SnapshotRecoveries []queryBuildSnapshotRecovery `bson:"snapshotRecoveries,omitempty"` Retired []queryBuildRetiredGeneration `bson:"retiredGenerations,omitempty"` ID string `bson:"_id"` Schema string `bson:"schema"` diff --git a/internal/store/query_build_process_restart_local_test.go b/internal/store/query_build_process_restart_local_test.go index 7952d03e..92096df2 100644 --- a/internal/store/query_build_process_restart_local_test.go +++ b/internal/store/query_build_process_restart_local_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "os/exec" + "sort" "sync/atomic" "testing" "time" @@ -134,6 +135,8 @@ func TestQueryBuildPhysicalMongoRestartMidApplyLocal(t *testing.T) { if e = json.Unmarshal(command("inspect", container), &after); e != nil || len(after) != 1 { t.Fatal(e) } + sort.Slice(cfg.Mounts, func(i, j int) bool { return cfg.Mounts[i].Destination < cfg.Mounts[j].Destination }) + sort.Slice(after[0].Mounts, func(i, j int) bool { return after[0].Mounts[i].Destination < after[0].Mounts[j].Destination }) a, _ := json.Marshal(cfg.Mounts) b, _ := json.Marshal(after[0].Mounts) if after[0].ID != cfg.ID || after[0].HostConfig.Memory != cfg.HostConfig.Memory || after[0].HostConfig.NanoCpus != cfg.HostConfig.NanoCpus || string(a) != string(b) { diff --git a/internal/store/query_build_resume_local_test.go b/internal/store/query_build_resume_local_test.go index 0e868902..18d8cfbc 100644 --- a/internal/store/query_build_resume_local_test.go +++ b/internal/store/query_build_resume_local_test.go @@ -425,3 +425,168 @@ func TestQueryBuildRetiredPayloadKeepsInflightReaderGraceLocal(t *testing.T) { t.Fatal("expired payload pinned forever", n, e) } } + +func TestQueryBuildExpiredUnsealedSnapshotRebasesOnlyCurrentGenerationLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + org := "workforce-test" + seedWorkforceStats(t, s) + s.queryReplicaFactStore = true + s.queryPeriodic = true + if e := s.ensureWorkforceFactIndexes(ctx); e != nil { + t.Fatal(e) + } + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil { + t.Fatal(e) + } + var before bson.M + if e = s.queryControl().FindOne(ctx, bson.M{"_id": "global"}).Decode(&before); e != nil { + t.Fatal(e) + } + if _, e = s.queryControl().UpdateOne(ctx, bson.M{"_id": "global"}, bson.M{"$inc": bson.M{"productionRevision": 1}}); e != nil { + t.Fatal(e) + } + j, e := s.queryBuildStart(ctx, l, org, []string{"p"}) + if e != nil { + t.Fatal(e) + } + j.Snapshot.T = 1 + j.FailureClass = "mongo_command" + j.FailureCode = 239 + j.FailurePhase = j.Phase + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil { + t.Fatal(e) + } + oldPart := queryBuildPart{ID: j.Generation + "/sentinel", Job: j.Generation, Data: []byte("retained old input")} + if _, e = s.db.Collection(queryBuildPartsCollection).InsertOne(ctx, oldPart); e != nil { + t.Fatal(e) + } + if _, e = s.queryData().InsertOne(ctx, bson.M{"_id": "superseded-blob", "generation": j.Generation + "-production", "createdAt": time.Now().Add(-24 * time.Hour)}); e != nil { + t.Fatal(e) + } + restarted := NewMongoStore(s.db) + restarted.queryReplicaFactStore = true + restarted.queryPeriodic = true + next, e := restarted.queryBuildStart(ctx, l, org, []string{"p"}) + if e != nil { + t.Fatal("expired unsealed input cannot recover", e) + } + if next.Generation == j.Generation || next.Snapshot.T == 1 || next.FailureClass != "" { + t.Fatal("recovery reused stale snapshot or failure fence") + } + if e = restarted.cleanQueryBuildParts(ctx, l); e != nil { + t.Fatal(e) + } + if e = restarted.cleanQueryModelOrphans(ctx); e != nil { + t.Fatal(e) + } + if n, e := s.db.Collection(queryBuildPartsCollection).CountDocuments(ctx, bson.M{"_id": oldPart.ID}); e != nil || n != 1 { + t.Fatal("unfinished recovery input discarded", n, e) + } + if n, e := s.queryData().CountDocuments(ctx, bson.M{"_id": "superseded-blob"}); e != nil || n != 1 { + t.Fatal("unfinished recovery blob discarded", n, e) + } + var during bson.M + if e = s.queryControl().FindOne(ctx, bson.M{"_id": "global"}).Decode(&during); e != nil { + t.Fatal(e) + } + if !reflect.DeepEqual(before["heads"], during["heads"]) { + t.Fatal("recovery replaced published cohort before completion") + } + if e = restarted.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil { + t.Fatal(e) + } + final, _, e := restarted.queryBuildJob(ctx, org) + if e != nil || final.Phase != "complete" || final.ConfirmedRows != 0 || final.ReusedRows == 0 { + t.Fatal("recovery rewrote unchanged facts", final.Phase, final.ConfirmedRows, final.ReusedRows, e) + } +} + +func TestQueryBuildExpiredSnapshotAutomaticallyYieldsAndRecoversLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + seedWorkforceStats(t, s) + s.queryReplicaFactStore = true + s.queryPeriodic = true + if e := s.ensureWorkforceFactIndexes(ctx); e != nil { + t.Fatal(e) + } + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + j, e := s.queryBuildStart(ctx, l, "workforce-test", []string{"p"}) + if e != nil { + t.Fatal(e) + } + j.Snapshot.T = 1 + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil { + t.Fatal(e) + } + if e = s.queryBuildRunRound(ctx, l, j.Organization, j.Projects, nil); !errors.Is(e, ErrQueryBuildYielded) || errors.Is(e, ErrQueryBuildStopped) { + t.Fatal("history loss stopped instead of scheduling replacement", e) + } + failed, _, e := s.queryBuildJob(ctx, j.Organization) + if e != nil || failed.FailureCode != 239 { + t.Fatal("original failure not retained", failed.FailureCode, e) + } + if e = s.queryBuildRunRound(ctx, l, j.Organization, j.Projects, nil); e != nil { + t.Fatal("replacement did not complete", e) + } + final, _, e := s.queryBuildJob(ctx, j.Organization) + if e != nil || final.Phase != "complete" || len(final.SnapshotRecoveries) != 1 { + t.Fatal("lost recovery provenance", e) + } +} + +func TestQueryBuildSnapshotRecoveryBoundaries(t *testing.T) { + for _, phase := range []string{"production_plan", "workforce_plan", "apply", "cleanup", "publish", "complete"} { + j := queryBuildJob{Phase: phase, FailureCode: 239, FailureClass: "mongo_command"} + want := phase == "production_plan" || phase == "workforce_plan" + if recoverableQueryBuildSnapshot(j) != want { + t.Fatal("wrong recovery phase", phase) + } + j.SnapshotRecoveries = make([]queryBuildSnapshotRecovery, 3) + if recoverableQueryBuildSnapshot(j) { + t.Fatal("unbounded recovery") + } + } +} + +func TestQueryBuildTransientResetIsFencedAndDoesNotFakeProgressLocal(t *testing.T) { + _, s, _ := newLocalReplica(t) + ctx := t.Context() + s.queryReplicaFactStore = true + s.queryPeriodic = true + l, ok, e := s.acquireQueryModelLease(ctx) + if e != nil || !ok { + t.Fatal(e) + } + j, e := s.queryBuildStart(ctx, l, "transient-test", nil) + if e != nil { + t.Fatal(e) + } + j.FailureClass = "network" + j.ProgressAt = time.Now().Add(-time.Minute) + j.ActiveWithoutProgress = time.Second + if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil { + t.Fatal(e) + } + if _, e = s.ResetTransientQueryBuildFailures(ctx); !errors.Is(e, ErrQueryReplicaLease) { + t.Fatal("reset bypassed active builder", e) + } + if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); e != nil { + t.Fatal(e) + } + if n, e := s.ResetTransientQueryBuildFailures(ctx); e != nil || n != 1 { + t.Fatal(n, e) + } + after, _, e := s.queryBuildJob(ctx, j.Organization) + if e != nil || after.FailureClass != "" || after.ActiveWithoutProgress != j.ActiveWithoutProgress || after.ProgressAt.UnixMilli() != j.ProgressAt.UnixMilli() { + t.Fatal("reset fabricated durable progress", e) + } +} diff --git a/internal/store/query_build_runner.go b/internal/store/query_build_runner.go index b9fda435..6b28acf7 100644 --- a/internal/store/query_build_runner.go +++ b/internal/store/query_build_runner.go @@ -25,15 +25,19 @@ func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org if err != nil { return old, err } + recoverSnapshot := exists && recoverableQueryBuildSnapshot(old) if exists && old.Phase != "complete" { - if old.FailureClass != "" { - return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass) - } if !reflect.DeepEqual(old.Projects, projects) { return old, errors.New("query build scope changed; retained job requires reconciliation") } - return old, nil + if !recoverSnapshot { + if old.FailureClass != "" { + return old, fmt.Errorf("%w: phase=%s category=%s", ErrQueryBuildStopped, old.FailurePhase, old.FailureClass) + } + return old, nil + } } + observedAt := time.Now().UTC() at, err := s.queryBuildSnapshot(ctx) if err != nil { @@ -59,6 +63,10 @@ func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()} j.Retired = append([]queryBuildRetiredGeneration{}, old.Retired...) j.InputGeneration = j.Generation + if recoverSnapshot { + j.SnapshotRecoveries = append(append([]queryBuildSnapshotRecovery{}, old.SnapshotRecoveries...), queryBuildSnapshotRecovery{Generation: old.Generation, Snapshot: old.Snapshot, AsOf: old.AsOf, Phase: old.Phase, Code: old.FailureCode, RecoveredAt: time.Now().UTC()}) + } + if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second { j.AsOf = state.Observed j.InputGeneration = old.InputGeneration @@ -114,7 +122,7 @@ func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, } else { current.ActiveWithoutProgress = 0 } - if errors.Is(err, ErrQueryBuildYielded) && current.ActiveWithoutProgress >= queryBuildNoProgressLimit { + if current.ActiveWithoutProgress >= queryBuildNoProgressLimit && (errors.Is(err, ErrQueryBuildYielded) || isTransientQueryBuildFailureClass(QueryReplicaDiagnostic(err).Class)) { err = ErrQueryBuildNoProgress } if !errors.Is(err, ErrQueryBuildYielded) { @@ -134,6 +142,11 @@ func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, log.Printf("query_model_build result=stopped stage=checkpoint category=%s mongoCode=%d", d.Class, d.MongoCode) err = fmt.Errorf("%w: query build could not persist outcome: %w", ErrQueryBuildStopped, e) } + if e == nil && recoverableQueryBuildSnapshot(current) { + log.Print("query_model_build result=yielded stage=input_snapshot category=expired checkpoint=retained") + err = fmt.Errorf("%w: input snapshot expired; replacement generation scheduled", ErrQueryBuildYielded) + } + }() for { if err = r.check(ctx); err != nil { @@ -164,7 +177,7 @@ func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, // Explicitly bounded derived-data housekeeping; active jobs are never swept. func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease) error { var jobs []queryBuildJob - cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1})) + cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"generation": 1, "inputGeneration": 1, "phase": 1, "snapshotRecoveries": 1})) if err != nil { return err } @@ -176,6 +189,11 @@ func (s *MongoStore) cleanQueryBuildParts(ctx context.Context, l queryModelLease keep := []string{} for _, j := range jobs { keep = append(keep, j.Generation, j.InputGeneration) + if j.Phase != "complete" { + for _, v := range j.SnapshotRecoveries { + keep = append(keep, v.Generation) + } + } } for _, collection := range []string{queryBuildPartsCollection, queryBuildJobsCollection} { f := bson.M{"job": bson.M{"$exists": true, "$nin": keep}} @@ -211,9 +229,6 @@ var transientQueryBuildFailureClasses = []string{ "timeout", "deadline", "network", - "lease_lost", - "lease", - "no_durable_progress", "transient", } @@ -232,6 +247,9 @@ func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int6 if s == nil || s.db == nil { return 0, errors.New("store unavailable") } + if !s.queryReplicaFactStore || !s.periodicQueryStore() { + return 0, ErrQueryReplicaUnsupported + } classes := bson.A{} for _, c := range transientQueryBuildFailureClasses { classes = append(classes, c) @@ -247,14 +265,33 @@ func (s *MongoStore) ResetTransientQueryBuildFailures(ctx context.Context) (int6 "failureClass": "", "failureCode": "", }, - "$set": bson.M{ - "activeWithoutProgress": int64(0), - "progressAt": time.Now().UTC(), - }, } - res, err := s.db.Collection(queryBuildJobsCollection).UpdateMany(ctx, filter, update) + lease, ok, err := s.acquireQueryModelLease(ctx) if err != nil { return 0, err } - return res.ModifiedCount, nil + if !ok { + return 0, ErrQueryReplicaLease + } + defer func() { + cleanup, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + _, _ = s.queryControl().UpdateOne(cleanup, queryLeaseFilter(lease), bson.M{"$set": bson.M{"owner": "", "until": time.Now().UTC()}}) + }() + var changed int64 + err = s.queryBuildTransaction(ctx, lease, func(sc mongo.SessionContext) error { + result, e := s.db.Collection(queryBuildJobsCollection).UpdateMany(sc, filter, update) + if e == nil { + changed = result.ModifiedCount + } + return e + }) + return changed, err +} + +// Only unfinished input capture can lose Mongo history. Sealed plans, commit +// failures, lease loss and corruption remain fenced. At most three replacements +// are attempted before reporting genuine inability to seal an input generation. +func recoverableQueryBuildSnapshot(j queryBuildJob) bool { + return j.FailureClass == "mongo_command" && j.FailureCode == 239 && (j.Phase == "production_plan" || j.Phase == "workforce_plan") && len(j.SnapshotRecoveries) < 3 } diff --git a/internal/store/query_build_transient_test.go b/internal/store/query_build_transient_test.go index 3513b222..e9d713fb 100644 --- a/internal/store/query_build_transient_test.go +++ b/internal/store/query_build_transient_test.go @@ -9,9 +9,6 @@ func TestIsTransientQueryBuildFailureClass(t *testing.T) { "timeout", "deadline", "network", - "lease_lost", - "lease", - "no_durable_progress", "transient", } { if !isTransientQueryBuildFailureClass(class) { @@ -23,6 +20,7 @@ func TestIsTransientQueryBuildFailureClass(t *testing.T) { "unsupported_record", "unsupported_fact", "corrupt_state", + "lease_lost", "lease", "no_durable_progress", "auth_failed", "bson_decode", } { diff --git a/internal/store/query_model_maintenance.go b/internal/store/query_model_maintenance.go index 4a73ae63..2e67de5a 100644 --- a/internal/store/query_model_maintenance.go +++ b/internal/store/query_model_maintenance.go @@ -42,7 +42,7 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error { keep = append(keep, h.Generation) } if s.queryReplicaFactStore && s.periodicQueryStore() { - cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"production.generation": 1, "phase": 1, "retiredGenerations": 1})) + cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"production.generation": 1, "phase": 1, "retiredGenerations": 1, "snapshotRecoveries": 1})) if err != nil { return err } @@ -53,6 +53,11 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error { return err } for _, j := range jobs { + if j.Phase != "complete" { + for _, r := range j.SnapshotRecoveries { + keep = append(keep, r.Generation+"-production") + } + } if j.Phase != "complete" && j.Production.Generation != "" { keep = append(keep, j.Production.Generation) } diff --git a/internal/store/store.go b/internal/store/store.go index cce519b6..ab600ea2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -365,6 +365,7 @@ type Store interface { DeleteUser(context.Context, string) (bool, error) UpsertUser(context.Context, domain.UserRecord) error UpdateUser(context.Context, domain.UserRecord, domain.UserRecord) error + AccountHasPendingWork(context.Context, string, string, domain.UserRole) (bool, error) CreateSession(context.Context, domain.SessionRecord) (*domain.SessionRecord, error) GetSessionByTokenHash(context.Context, string) (*domain.SessionRecord, error) DeleteSession(context.Context, string) (bool, error) diff --git a/scripts/local/evening-query-cutover.py b/scripts/local/evening-query-cutover.py index 6ce1d66e..ead2d36e 100644 --- a/scripts/local/evening-query-cutover.py +++ b/scripts/local/evening-query-cutover.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import datetime as dt import json +import hashlib import os from pathlib import Path import re @@ -13,6 +14,8 @@ import subprocess import sys import time from typing import Any +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from query_cutover_guard import approved_plan, bounded_shadow_args, require_result_comparison DEFAULT_KUBECONFIG = "/Users/d-robotics/Desktop/fanyu/droboticflow/local-data/fanyu.conf" SOURCE_NAMESPACE = "droboticflow-fanyu" @@ -102,6 +105,8 @@ def redact_secrets(obj: Any, parent_key: str = "") -> Any: return {k: redact_secrets(v, k) for k, v in obj.items()} if isinstance(obj, list): return [redact_secrets(item, parent_key) for item in obj] + if isinstance(obj, str) and obj.startswith("--resume-pause="): + return "--resume-pause=" if isinstance(obj, str) and parent_key and SECRET_KEY_PATTERN.search(parent_key): return "" return obj @@ -146,9 +151,13 @@ def build_deployment_json_patch( strategy_recreate: bool = False, image: str | None = None, args: list[str] | None = None, + expected_deployment: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """Build an atomic RFC 6902 JSON patch for a single deployment container.""" ops: list[dict[str, Any]] = [] + if expected_deployment: + ops.extend([{"op":"test", "path":"/metadata/uid", "value":expected_deployment["metadata"]["uid"]}, + {"op":"test", "path":"/metadata/resourceVersion", "value":expected_deployment["metadata"]["resourceVersion"]}]) if strategy_recreate: ops.append({"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}}) ops.append( @@ -177,14 +186,8 @@ def build_deployment_json_patch( return ops -def build_shadow_replica_args() -> list[str]: - return [ - "--writers-covered", - "--bootstrap", - "--source-compress", - "--batch-size=100", - "--interval=30s", - ] +def build_shadow_replica_args(plan, resume_nonce="") -> list[str]: + return bounded_shadow_args(plan, resume_nonce) def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -198,11 +201,8 @@ def build_shadow_replica_env(existing_env: list[dict[str, Any]]) -> list[dict[st return merge_env_list(existing_env, updates) -def build_shadow_builder_args() -> list[str]: - return [ - "--ensure-indexes", - "--auto-recover-transient", - ] +def build_shadow_builder_args(plan) -> list[str]: + return ["--ensure-indexes", "--shadow-safety-run=" + plan["shadowPolicy"]["runID"]] def build_shadow_builder_env(existing_env: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -367,6 +367,25 @@ def evaluate_parity_snapshot( if wf_age < -5 or wf_age >= max_workforce_age_seconds: reasons.append(f"workforce asOf age out of bounds: {wf_age:.1f}s") + cohorts = snapshot.get("cohorts") or [] + if not cohorts: + reasons.append("scoped publication cohort identities missing") + heads = (snapshot.get("global") or {}).get("heads") or {} + for cohort in cohorts: + production = heads.get(cohort.get("productionKey")) or {} + workforce = heads.get(cohort.get("workforceKey")) or {} + if not production.get("generation") or parse_bson_int(production.get("chunks")) < 1 or workforce.get("generation") != "workforce-facts/v1": + reasons.append("scoped publication incomplete") + continue + for field in ("asOf", "sourceRevision", "globalRevision"): + if production.get(field) is None or production.get(field) != workforce.get(field): + reasons.append("mixed scoped publication cohort") + break + for head in (production, workforce): + at = parse_iso_datetime(head.get("asOf")) + if at is None or not 0 <= (current_time-at).total_seconds() < max_workforce_age_seconds: + reasons.append("scoped publication stale") + if parse_bson_int(snapshot.get("poolBucketsCount")) <= 0: reasons.append("queryPoolCountBuckets is empty") if parse_bson_int(snapshot.get("workforceFactsCount")) <= 0: @@ -384,6 +403,7 @@ class ClusterClient: def __init__(self, kubeconfig: str) -> None: self.kubeconfig = kubeconfig + self.approved_plan = None def run( self, @@ -415,27 +435,35 @@ class ClusterClient: args.extend(["get", *resource_args, "-o", "json"]) if ignore_not_found: args.append("--ignore-not-found") - proc = self.run(args) - if ignore_not_found and not proc.stdout.strip(): - return None - return json.loads(proc.stdout) + for attempt in range(3): + try: + proc = self.run(args, timeout=8, check=False) + if proc.returncode == 0: + if ignore_not_found and not proc.stdout.strip(): + return None + return json.loads(proc.stdout) + if any(term in proc.stderr for term in ("Forbidden", "Unauthorized", "NotFound")): + raise RuntimeError("metadata authorization/identity failure") + except (subprocess.TimeoutExpired, OSError): + pass + print(json.dumps({"event":"metadata_probe_retry","attempt":attempt+1}),file=sys.stderr) + if attempt < 2: + time.sleep(.5 * (attempt+1)) + raise RuntimeError("metadata probe unavailable after bounded retries") def patch_json_ops(self, namespace: str, kind: str, name: str, ops: list[dict[str, Any]]) -> None: - self.run( - [ - "-n", - namespace, - "patch", - kind, - name, - "--type=json", - "-p", - json.dumps(ops), - ] - ) + if not self.approved_plan: + raise RuntimeError("approved plan required before cluster writes") + current = self.get_json(namespace, kind, name) + expected = self.approved_plan["resourceUIDs"].get(namespace + "/" + kind + "/" + name) + if not expected or current["metadata"]["uid"] != expected: + raise RuntimeError("resource identity outside approved plan") + tests = [{"op":"test", "path":"/metadata/uid", "value":expected}, + {"op":"test", "path":"/metadata/resourceVersion", "value":current["metadata"]["resourceVersion"]}] + self.run(["-n",namespace,"patch",kind,name,"--type=json","--patch-file=/dev/stdin"], stdin_text=json.dumps(tests + ops)) def scale(self, namespace: str, kind: str, name: str, replicas: int) -> None: - self.run(["-n", namespace, "scale", f"{kind}/{name}", f"--replicas={replicas}"]) + self.patch_json_ops(namespace, kind, name, [{"op":"replace", "path":"/spec/replicas", "value":replicas}]) def rollout_status(self, namespace: str, deployment: str, timeout_seconds: int = 180) -> bool: proc = self.run( @@ -464,6 +492,12 @@ if (q.auth('shadow-health', fs.readFileSync('/health-credentials/health-password quit(2); }} const st = q.queryReplicaControl.findOne({{_id: 'state'}}); +const crypto = require('crypto'); +const modelKey = (kind, org, projects) => 'query-model/v3:' + crypto.createHash('sha256').update(JSON.stringify([kind, org, projects])).digest('hex'); +const cohorts = q.productionStatsScopes.find({{}}, {{projectIds:1}}).toArray().map(s => {{ + const ids=q.projects.find({{id:{{$in:s.projectIds||[]}}}}, {{id:1}}).toArray().map(p=>p.id).sort(); + return {{productionKey:modelKey('production',String(s._id).trim(),ids),workforceKey:modelKey('workforce-facts',String(s._id).trim(),[]) ,projects:ids.length}}; +}}).filter(c=>c.projects>0); const gl = q.queryReadModelControl.findOne({{_id: 'global'}}); const bootDone = q.queryReplicaControl.countDocuments({{_id: /^bootstrap\\//, complete: true}}); const poolBucketsCount = q.queryPoolCountBuckets.countDocuments({{}}); @@ -490,6 +524,7 @@ print(JSON.stringify({{ factEpoch: Number(st.factEpoch || 0), schema: st.schema || null }} : null, + cohorts: cohorts, bootstrapCompleteCount: bootDone, poolBucketsCount: poolBucketsCount, workforceFactsCount: workforceFactsCount, @@ -606,23 +641,38 @@ def cmd_preflight(args: argparse.Namespace, client: ClusterClient, out_dir: Path return 0 +def read_shadow_safety(client): + js = "const fs=require('fs');const q=db.getSiblingDB('" + QUERY_DB_NAME + "');if(q.auth('shadow-health',fs.readFileSync('/health-credentials/health-password','utf8').trim()).ok!==1)quit(2);print(EJSON.stringify({safety:q.queryReplicaControl.findOne({_id:'safety'}),health:q.queryReplicaControl.findOne({_id:'safety-health'})},{relaxed:true}));" + p = client.run(["-n",SHADOW_NAMESPACE,"exec",SHADOW_MONGO_POD,"-c",SHADOW_MONGO_CONTAINER,"--","mongosh","--quiet","--host","127.0.0.1","--port","27129","--eval",js], timeout=25) + return json.loads(p.stdout.strip().splitlines()[-1]) + + def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: baseline_health = inspect_api_health(client) if baseline_health["readyReplicas"] < baseline_health["desiredReplicas"] or baseline_health["desiredReplicas"] == 0: raise RuntimeError(f"baseline API unhealthy before shadow-sync: {baseline_health}") - # Disarm legacy shadow monitor if running so its strict safety gate cannot scale controllers back to 0. - mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True) - if mon_dep is not None and int(mon_dep.get("spec", {}).get("replicas") or 0) > 0: - client.scale(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, 0) - mon_deadline = time.monotonic() + 60 - while time.monotonic() < mon_deadline: - d = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT, ignore_not_found=True) - if d is None or int(d.get("status", {}).get("replicas") or 0) == 0: - break - time.sleep(2) - else: - raise RuntimeError(f"timed out waiting for {MONITOR_DEPLOYMENT} to scale to zero") + mon_dep = client.get_json(SHADOW_NAMESPACE, "deployment", MONITOR_DEPLOYMENT) + if int(mon_dep.get("status", {}).get("availableReplicas") or 0) != 1: + raise RuntimeError("approved shadow observer must be healthy; it is never disabled by synchronization") + policy = client.approved_plan["shadowPolicy"] + safety = read_shadow_safety(client) + health = safety.get("health") or {} + recent = health.get("recent", [])[-2:] + if len(recent) != 2 or any(not h.get("ready") or not h.get("sourceAvailable") or h.get("workErrors") or h.get("memoryPressure") or h.get("persistenceFailure") for h in recent): + raise RuntimeError("two fresh complete health observations required") + for h in recent: + at = parse_iso_datetime(h.get("at")) + if at is None or not 0 <= (utc_now()-at).total_seconds() <= 60: + raise RuntimeError("shadow health evidence is stale") + saved = safety.get("safety") or {} + if (saved.get("policy") or {}).get("runId") != policy["runID"]: + raise RuntimeError("synchronization run changed; cannot reset checkpoint/budget") + if int(saved.get("reserved") or 0) >= policy["maxRecords"]: + raise RuntimeError("approved cumulative budget exhausted") + if saved.get("paused") and saved.get("reason") not in policy.get("resumeReasons", ["operator_stop", "pilot_budget", "maintenance_window_closed"]): + raise RuntimeError("unexpected hard pause requires diagnosis, not automatic reset") + nonce = saved.get("resumeToken", "") if saved.get("paused") else "" # Scale builder to 0 first to stay well inside ResourceQuota. client.scale(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, 0) @@ -635,9 +685,10 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa rep_env = rep_dep["spec"]["template"]["spec"]["containers"][rep_idx].get("env") or [] rep_patch = build_deployment_json_patch( container_index=rep_idx, + expected_deployment=rep_dep, strategy_recreate=True, image=args.replica_image, - args=build_shadow_replica_args(), + args=build_shadow_replica_args(client.approved_plan, nonce), env_list=build_shadow_replica_env(rep_env), ) client.patch_json_ops(SHADOW_NAMESPACE, "deployment", REPLICA_DEPLOYMENT, rep_patch) @@ -675,9 +726,10 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa bld_env = bld_dep["spec"]["template"]["spec"]["containers"][bld_idx].get("env") or [] bld_patch = build_deployment_json_patch( container_index=bld_idx, + expected_deployment=bld_dep, strategy_recreate=True, image=args.builder_image, - args=build_shadow_builder_args(), + args=build_shadow_builder_args(client.approved_plan), env_list=build_shadow_builder_env(bld_env), ) client.patch_json_ops(SHADOW_NAMESPACE, "deployment", BUILDER_DEPLOYMENT, bld_patch) @@ -715,6 +767,15 @@ def cmd_shadow_sync(args: argparse.Namespace, client: ClusterClient, out_dir: Pa def cmd_verify_parity(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: snap = query_shadow_mongo_snapshot(client) ok, reasons = evaluate_parity_snapshot(snap) + if ok: + try: + if not getattr(args, "comparison_evidence", ""): + raise RuntimeError("result/permission comparison evidence missing") + plan = client.approved_plan or json.loads(Path(args.approved_plan).read_text()) + require_result_comparison(json.loads(Path(args.comparison_evidence).read_text()), snap, plan) + except (RuntimeError, OSError, ValueError, KeyError) as error: + ok = False + reasons.append(str(error)) report = { "at": utc_now().isoformat(), "passed": ok, @@ -735,8 +796,13 @@ def sync_read_client_secret(client: ClusterClient) -> None: raise RuntimeError("required shadow read client secret keys missing") manifest = build_read_client_secret_manifest(api_uri_b64, valkey_url_b64) existing = client.get_json(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, ignore_not_found=True) - verb = "replace" if existing is not None else "create" - client.run([verb, "-f", "-"], stdin_text=json.dumps(manifest)) + if existing is not None: + client.patch_json_ops(SOURCE_NAMESPACE, "secret", READ_CLIENT_SECRET_NAME, [{"op":"replace", "path":"/data", "value":manifest["data"]}]) + else: + resource = SOURCE_NAMESPACE + "/secret/" + READ_CLIENT_SECRET_NAME + if resource not in client.approved_plan.get("createResources", []): + raise RuntimeError("read client Secret creation not authorized") + client.run(["create", "-f", "-"], stdin_text=json.dumps(manifest)) def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_dir: Path) -> int: @@ -744,7 +810,7 @@ def cmd_rollback_api_flags(args: argparse.Namespace, client: ClusterClient, out_ api_idx = find_container_index(api_dep, API_CONTAINER) existing_env = api_dep["spec"]["template"]["spec"]["containers"][api_idx].get("env") or [] rolled_env = merge_env_list(existing_env, build_api_rollback_env_updates()) - patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env) + patch = build_deployment_json_patch(container_index=api_idx, env_list=rolled_env, expected_deployment=api_dep) client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) ok = client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180) health = inspect_api_health(client) @@ -763,6 +829,9 @@ def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_d if not ok: raise RuntimeError(f"parity gate blocked cutover: {reasons}") + if not getattr(args, "comparison_evidence", ""): + raise RuntimeError("new/old result and permission evidence required before production cutover") + require_result_comparison(json.loads(Path(args.comparison_evidence).read_text()), snap, client.approved_plan) sync_read_client_secret(client) api_dep = client.get_json(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT) api_idx = find_container_index(api_dep, API_CONTAINER) @@ -770,7 +839,7 @@ def cmd_cutover_api_flags(args: argparse.Namespace, client: ClusterClient, out_d write_report(out_dir, "api-env-before.json", {"at": utc_now().isoformat(), "env": existing_env}) next_env = merge_env_list(existing_env, build_api_cutover_env_updates()) - patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env) + patch = build_deployment_json_patch(container_index=api_idx, env_list=next_env, expected_deployment=api_dep) client.patch_json_ops(SOURCE_NAMESPACE, "deployment", API_DEPLOYMENT, patch) if not client.rollout_status(SOURCE_NAMESPACE, API_DEPLOYMENT, timeout_seconds=180): @@ -802,7 +871,9 @@ def cmd_watch_and_guard(args: argparse.Namespace, client: ClusterClient, out_dir timeout=15, check=False, ) - error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "") + if log_proc.returncode != 0: + raise RuntimeError("API health log probe unavailable; no zero-error claim") + error_lines = count_error_log_lines(log_proc.stdout) sample = { "at": utc_now().isoformat(), "readyReplicas": health["readyReplicas"], @@ -875,7 +946,9 @@ def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) - timeout=15, check=False, ) - error_lines = count_error_log_lines(log_proc.stdout if log_proc.returncode == 0 else "") + if log_proc.returncode != 0: + raise RuntimeError("API health log probe unavailable; no zero-error claim") + error_lines = count_error_log_lines(log_proc.stdout) snap = None parity_ok = False @@ -943,6 +1016,10 @@ def cmd_patrol(args: argparse.Namespace, client: ClusterClient, out_dir: Path) - def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Unattended evening query offload cutover and guard.") parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG) + parser.add_argument("--approved-plan", default="") + parser.add_argument("--confirm-plan", default="") + parser.add_argument("--approval-ref", default="") + parser.add_argument("--comparison-evidence", default="") parser.add_argument("--run-id", default=default_run_id()) parser.add_argument("--output-dir", default="") @@ -979,6 +1056,7 @@ def main(argv: list[str] | None = None) -> int: else Path(__file__).resolve().parents[2] / "local-data" / "diagnostics" / "evening-cutover" / args.run_id ) client = ClusterClient(args.kubeconfig) + client.approved_plan = approved_plan(args, client) handlers = { "preflight": cmd_preflight, "shadow-sync": cmd_shadow_sync, @@ -988,7 +1066,18 @@ def main(argv: list[str] | None = None) -> int: "patrol": cmd_patrol, "rollback-api-flags": cmd_rollback_api_flags, } - return handlers[args.command](args, client, out_dir) + try: + return handlers[args.command](args, client, out_dir) + except Exception: + if args.command == "shadow-sync" and client.approved_plan: + failures = [] + for name in (REPLICA_DEPLOYMENT, BUILDER_DEPLOYMENT): + try: + client.scale(SHADOW_NAMESPACE, "deployment", name, 0) + except Exception as error: + failures.append({"controller":name, "errorClass":type(error).__name__}) + write_report(out_dir, "shadow-sync-stopped.json", {"progressRetained":True,"stopFailures":failures}) + raise if __name__ == "__main__": diff --git a/scripts/local/query_cutover_guard.py b/scripts/local/query_cutover_guard.py new file mode 100644 index 00000000..8940c2f7 --- /dev/null +++ b/scripts/local/query_cutover_guard.py @@ -0,0 +1,102 @@ +"""Small, explicit authorization and result-evidence checks for query cutover. + +No cluster writes, telemetry or production fallback at import time. +""" +import datetime as dt +import hashlib +import json +import subprocess +from pathlib import Path +MUTATING_COMMANDS = {'shadow-sync', 'cutover-api-flags', 'rollback-api-flags', 'watch-and-guard', 'patrol'} +ROLES = {'admin', 'supplier_manager', 'annotator', 'quality_reviewer', 'acceptance_reviewer'} +SCENARIOS = {'normal', 'legacy', 'date-boundary', 'deleted', 'rework', 'permission-revoked', 'cross-supplier', 'cross-organization'} + +def utc(value): + return dt.datetime.fromisoformat(value.replace('Z', '+00:00')) + +def approved_plan(args, client): + if args.command not in MUTATING_COMMANDS: + return None + path = getattr(args, 'approved_plan', None) + if not path or not getattr(args, 'approval_ref', ''): + raise RuntimeError('explicit approved plan and approval reference required; no writes performed') + raw = Path(path).read_bytes() + if hashlib.sha256(raw).hexdigest() != getattr(args, 'confirm_plan', ''): + raise RuntimeError('approved plan digest changed') + plan = json.loads(raw) + if args.command not in plan.get('operations', []) or dt.datetime.now(dt.timezone.utc) >= utc(plan['expiresAt']): + raise RuntimeError('operation outside approved scope/window') + if args.command == 'cutover-api-flags' and plan.get('allowProductionCutover') is not True: + raise RuntimeError('production entry is not authorized') + refs = [subprocess.check_output(['git', 'rev-parse', ref], text=True).strip() for ref in ('HEAD', 'dev', 'origin/dev')] + if not all((ref == plan.get('sourceCommit') for ref in refs)) or subprocess.check_output(['git', 'status', '--porcelain'], text=True).strip(): + raise RuntimeError('clean synchronized release source required') + remote = subprocess.check_output(['git','ls-remote','--exit-code','origin','refs/heads/dev'],text=True,stderr=subprocess.DEVNULL).split()[0] + if remote != plan.get('sourceCommit'): + raise RuntimeError('remote dev changed since approved plan') + context = client.run(['config', 'current-context']).stdout.strip() + if context != plan['context']: + raise RuntimeError('cluster context changed') + server = client.run(['config', 'view', '--minify', '-o', 'jsonpath={.clusters[0].cluster.server}']).stdout.strip() + if hashlib.sha256(server.encode()).hexdigest() != plan['serverSHA256']: + raise RuntimeError('cluster endpoint changed') + for namespace, uid in plan['namespaceUIDs'].items(): + if client.get_json(None, 'namespace', namespace)['metadata']['uid'] != uid: + raise RuntimeError('namespace identity changed') + if args.command == 'shadow-sync': + for field in ('replica_image', 'builder_image'): + if getattr(args, field) != plan.get(field) or '@sha256:' not in getattr(args, field): + raise RuntimeError('immutable candidate image differs from approved plan') + return plan + +def bounded_shadow_args(plan, resume_nonce=''): + policy = plan['shadowPolicy'] + if not policy.get('runID') or not all((type(policy.get(k)) is int and policy[k] > 0 for k in ('maxRecords', 'recordsPerSecond', 'bytesPerSecond', 'commandsPerSecond', 'batchSize'))): + raise RuntimeError('explicit finite shadow budgets required') + if utc(policy['maintenanceUntil']) > utc(plan['expiresAt']): + raise RuntimeError('shadow policy exceeds approved plan') + if dt.datetime.now(dt.timezone.utc) >= utc(policy['maintenanceUntil']): + raise RuntimeError('shadow maintenance window closed') + args = ['--writers-covered', '--bootstrap', '--interval=30s', '--maintenance-initialization', '--safety-run=' + policy['runID'], '--maintenance-until=' + policy['maintenanceUntil'], '--max-source-records=' + str(policy['maxRecords']), '--source-records-per-second=' + str(policy['recordsPerSecond']), '--source-bytes-per-second=' + str(policy['bytesPerSecond']), '--source-commands-per-second=' + str(policy['commandsPerSecond']), '--batch-size=' + str(policy['batchSize'])] + if resume_nonce: + args.append('--resume-pause=' + resume_nonce) + return args + +def require_result_comparison(evidence, snapshot, plan, now=None): + now = now or dt.datetime.now(dt.timezone.utc) + if evidence.get('environment') != 'shadow-live' or evidence.get('queryDatabase') != 'drobodata_fanyu_query_shadow_20261114': + raise RuntimeError('live scoped comparison evidence required') + if evidence.get('sourceCommit') != plan.get('sourceCommit') or evidence.get('namespaceUID') != plan['namespaceUIDs']['droboticflow-query-shadow']: + raise RuntimeError('comparison identity mismatch') + age = (now - utc(evidence['observedAt'])).total_seconds() + if not 0 <= age <= 900: + raise RuntimeError('comparison evidence expired') + if evidence.get('cohortDigest') != hashlib.sha256(json.dumps(snapshot['global']['heads'], sort_keys=True, separators=(',', ':')).encode()).hexdigest(): + raise RuntimeError('published comparison cohort changed') + if evidence.get('replicaFactEpoch') != snapshot.get('state', {}).get('factEpoch'): + raise RuntimeError('comparison replica epoch changed') + cases = evidence.get('cases') or [] + for case in cases: + old, new = (case.get('authorityDigest', ''), case.get('queryDigest', '')) + if len(old) != 64 or old != new or any((c not in '0123456789abcdef' for c in old)): + raise RuntimeError('result comparison differs or is missing') + if case.get('authorityAllowed'): + heads = snapshot['global']['heads'] + production = heads.get(case.get('productionHeadKey')) or {} + workforce = heads.get(case.get('workforceHeadKey')) or {} + if workforce.get('generation') != 'workforce-facts/v1' or not production.get('chunks'): + raise RuntimeError('scoped publication heads missing') + for field in ('asOf', 'sourceRevision', 'globalRevision'): + if production.get(field) is None or production[field] != workforce.get(field): + raise RuntimeError('mixed publication cohort') + if type(case.get('authorityAllowed')) is not bool or case['authorityAllowed'] != case.get('queryAllowed'): + raise RuntimeError('permission comparison differs') + if case.get('authorityScopeDigest') != case.get('queryScopeDigest') or not case.get('authorityScopeDigest'): + raise RuntimeError('organization/supplier scope comparison differs') + if not any((c.get('authorityAllowed') for c in cases)): + raise RuntimeError('successful business results not compared') + for scenario in ('permission-revoked', 'cross-supplier', 'cross-organization'): + if not any((c.get('scenario') == scenario and c.get('authorityAllowed') is False for c in cases)): + raise RuntimeError('permission denial coverage missing') + if not ROLES <= {c.get('role') for c in cases} or not SCENARIOS <= {c.get('scenario') for c in cases} or (not {'counts', 'dashboard', 'work-records'} <= {c.get('kind') for c in cases}): + raise RuntimeError('comparison coverage incomplete') diff --git a/scripts/local/test_evening_query_cutover.py b/scripts/local/test_evening_query_cutover.py index 6eb5f4a3..130f4a99 100644 --- a/scripts/local/test_evening_query_cutover.py +++ b/scripts/local/test_evening_query_cutover.py @@ -3,6 +3,8 @@ import importlib.util import json from pathlib import Path import unittest +import tempfile +from types import SimpleNamespace def load_module(): @@ -96,14 +98,14 @@ class EveningQueryCutoverTests(unittest.TestCase): container_index=1, strategy_recreate=True, image="example/img@sha256:abc", - args=CUTOVER.build_shadow_replica_args(), + args=CUTOVER.build_shadow_replica_args({"expiresAt":"2099-01-01T00:00:00Z","shadowPolicy":{"runID":"same-run","maintenanceUntil":"2099-01-01T00:00:00Z","maxRecords":15000000,"recordsPerSecond":100,"bytesPerSecond":1048576,"commandsPerSecond":16,"batchSize":100}}), env_list=CUTOVER.build_shadow_replica_env([]), ) by_path = {op["path"]: op for op in patch_ops} self.assertEqual(by_path["/spec/strategy"], {"op": "add", "path": "/spec/strategy", "value": {"type": "Recreate"}}) self.assertEqual(by_path["/spec/template/spec/containers/1/image"]["value"], "example/img@sha256:abc") - self.assertIn("--source-compress", by_path["/spec/template/spec/containers/1/args"]["value"]) - self.assertNotIn("--safety-run", by_path["/spec/template/spec/containers/1/args"]["value"]) + self.assertIn("--maintenance-initialization", by_path["/spec/template/spec/containers/1/args"]["value"]) + self.assertIn("--safety-run=same-run", by_path["/spec/template/spec/containers/1/args"]["value"]) def test_evaluate_parity_snapshot_gates_ready_fresh_derived_data(self): now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) @@ -111,10 +113,11 @@ class EveningQueryCutoverTests(unittest.TestCase): "state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z", "factEpoch": 3}, "poolBucketsCount": 42, "workforceFactsCount": 1200, + "cohorts":[{"productionKey":"h1","workforceKey":"h2"}], "global": { "heads": { - "h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"}, - "h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"}, + "h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:57:00Z"}, + "h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:57:00Z"}, } }, } @@ -127,7 +130,7 @@ class EveningQueryCutoverTests(unittest.TestCase): stale["workforceFactsCount"] = 0 passed, reasons = CUTOVER.evaluate_parity_snapshot(stale, now=now) self.assertFalse(passed) - self.assertEqual(len(reasons), 2) + self.assertGreaterEqual(len(reasons), 2) def test_proactive_patrol_thresholds_trip_before_15min_expiry(self): now = dt.datetime(2026, 11, 16, 12, 0, 0, tzinfo=dt.timezone.utc) @@ -135,10 +138,11 @@ class EveningQueryCutoverTests(unittest.TestCase): "state": {"ready": True, "observedAt": "2026-11-16T11:49:30Z", "factEpoch": 3}, "poolBucketsCount": 42, "workforceFactsCount": 1200, + "cohorts":[{"productionKey":"h1","workforceKey":"h2"}], "global": { "heads": { - "h1": {"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"}, - "h2": {"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"}, + "h1": {"sourceRevision":1,"globalRevision":1,"generation": "prod-gen-1", "chunks": 2, "asOf": "2026-11-16T11:47:30Z"}, + "h2": {"sourceRevision":1,"globalRevision":1,"generation": "workforce-facts/v1", "factEpoch": 2, "asOf": "2026-11-16T11:47:30Z"}, } }, } @@ -149,7 +153,7 @@ class EveningQueryCutoverTests(unittest.TestCase): max_workforce_age_seconds=720, ) self.assertFalse(passed) - self.assertEqual(len(reasons), 2) + self.assertGreaterEqual(len(reasons), 2) self.assertTrue(CUTOVER.api_cutover_active(CUTOVER.build_api_cutover_env_updates())) self.assertFalse(CUTOVER.api_cutover_active(CUTOVER.build_api_rollback_env_updates())) @@ -164,14 +168,17 @@ class EveningQueryCutoverTests(unittest.TestCase): "state": {"ready": True, "observedAt": "2026-11-16T11:58:00Z"}, "poolBucketsCount": {"high": 0, "low": 28730, "unsigned": False}, "workforceFactsCount": {"high": 0, "low": 140082, "unsigned": False}, + "cohorts":[{"productionKey":"h1","workforceKey":"h2"}], "global": { "heads": { "h1": { + "sourceRevision":1,"globalRevision":1, "generation": "prod-gen-1", "chunks": {"high": 0, "low": 45, "unsigned": False}, "asOf": "2026-11-16T11:57:00Z", }, "h2": { + "sourceRevision":1,"globalRevision":1, "generation": "workforce-facts/v1", "factEpoch": {"high": 0, "low": 1, "unsigned": False}, "asOf": "2026-11-16T11:57:00Z", @@ -199,5 +206,65 @@ class EveningQueryCutoverTests(unittest.TestCase): self.assertEqual(by_name[key]["value"], "0") + +class CutoverMutationProtectionTests(unittest.TestCase): + def test_patch_uses_uid_rv_and_stdin(self): + from unittest.mock import Mock + client = CUTOVER.ClusterClient("unused") + client.approved_plan = {"resourceUIDs": {"ns/deployment/app": "uid"}} + client.get_json = Mock(return_value={"metadata": {"uid": "uid", "resourceVersion": "10"}}) + client.run = Mock() + client.patch_json_ops("ns", "deployment", "app", [{"op": "replace", "path": "/spec/replicas", "value": 0}]) + argv = client.run.call_args.args[0] + ops = json.loads(client.run.call_args.kwargs["stdin_text"]) + self.assertIn("--patch-file=/dev/stdin", argv) + self.assertEqual([o["path"] for o in ops[:2]], ["/metadata/uid", "/metadata/resourceVersion"]) + self.assertEqual(ops[1]["value"], "10") + client.get_json.return_value["metadata"]["uid"] = "replaced" + with self.assertRaises(RuntimeError): + client.scale("ns", "deployment", "app", 1) + + def test_original_read_version_is_preserved_in_patch(self): + dep = {"metadata": {"uid": "uid", "resourceVersion": "9"}} + ops = CUTOVER.build_deployment_json_patch(env_list=[], expected_deployment=dep) + self.assertEqual(ops[1], {"op": "test", "path": "/metadata/resourceVersion", "value": "9"}) + + def test_observer_not_disarmed_to_force_sync(self): + from unittest.mock import Mock, patch + client = Mock() + client.get_json.return_value = {"status": {"availableReplicas": 0}} + with tempfile.TemporaryDirectory() as directory, patch.object(CUTOVER,"inspect_api_health",return_value={"readyReplicas":4,"desiredReplicas":4}): + with self.assertRaisesRegex(RuntimeError,"observer"): + CUTOVER.cmd_shadow_sync(SimpleNamespace(),client,Path(directory)) + client.scale.assert_not_called() + + def test_nonce_never_saved_in_snapshot_reports(self): + self.assertEqual(CUTOVER.redact_secrets({"args":["--resume-pause=private-token"]}),{"args":["--resume-pause="]}) + +class ScopedPublicationTests(unittest.TestCase): + def test_unrelated_or_mixed_heads_never_mean_scope_ready(self): + now=dt.datetime(2026,11,16,12,tzinfo=dt.timezone.utc) + base={"state":{"ready":True,"observedAt":now.isoformat()},"poolBucketsCount":1,"workforceFactsCount":1,"cohorts":[{"productionKey":"p","workforceKey":"w"}],"global":{"heads":{"p":{"generation":"p","chunks":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2},"w":{"generation":"workforce-facts/v1","factEpoch":1,"asOf":now.isoformat(),"sourceRevision":3,"globalRevision":2}}}} + self.assertTrue(CUTOVER.evaluate_parity_snapshot(base,now=now)[0]) + for key in ("asOf","sourceRevision","globalRevision"): + mixed=json.loads(json.dumps(base));mixed['global']['heads']['w'][key]=None + self.assertFalse(CUTOVER.evaluate_parity_snapshot(mixed,now=now)[0]) + wrong=json.loads(json.dumps(base));wrong['cohorts'][0]['productionKey']='other-org' + self.assertFalse(CUTOVER.evaluate_parity_snapshot(wrong,now=now)[0]) + +class BoundedMetadataProbeTests(unittest.TestCase): + def test_transient_metadata_failure_retries_but_auth_failure_does_not(self): + from unittest.mock import Mock, patch + import subprocess + client=CUTOVER.ClusterClient('unused') + client.run=Mock(side_effect=[subprocess.TimeoutExpired('get',8),SimpleNamespace(returncode=1,stderr='temporary',stdout=''),SimpleNamespace(returncode=0,stdout='{"metadata":{"uid":"u"}}')]) + with patch.object(CUTOVER.time,'sleep'): + self.assertEqual(client.get_json('ns','deployment','app')['metadata']['uid'],'u') + self.assertEqual(client.run.call_count,3) + client.run=Mock(return_value=SimpleNamespace(returncode=1,stderr='Forbidden',stdout='')) + with self.assertRaises(RuntimeError):client.get_json('ns','deployment','app') + self.assertEqual(client.run.call_count,1) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/local/test_query_cutover_guard.py b/scripts/local/test_query_cutover_guard.py new file mode 100644 index 00000000..93ab2315 --- /dev/null +++ b/scripts/local/test_query_cutover_guard.py @@ -0,0 +1,83 @@ +import datetime as dt +import hashlib +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +sys.path.insert(0, str(Path(__file__).parent)) +import query_cutover_guard as guard + +class CutoverGuardTests(unittest.TestCase): + + def plan(self): + return {'shadowPolicy': {'runID': 'same-run', 'maxRecords': 15000000, 'recordsPerSecond': 100, 'bytesPerSecond': 1048576, 'commandsPerSecond': 16, 'batchSize': 100, 'maintenanceUntil': '2099-01-01T00:00:00Z'}, 'expiresAt': '2099-01-01T00:00:00Z', 'sourceCommit': 'candidate', 'namespaceUIDs': {'droboticflow-query-shadow': 'shadow-uid'}} + + def evidence(self): + snap = {'state': {'factEpoch': 3}, 'global': {'heads': {'p': {'generation': 'p-gen', 'chunks': 1, 'asOf': '2026-11-16T12:00:00Z', 'sourceRevision': 4, 'globalRevision': 2}, 'w': {'generation': 'workforce-facts/v1', 'factEpoch': 1, 'asOf': '2026-11-16T12:00:00Z', 'sourceRevision': 4, 'globalRevision': 2}}}} + cases = [] + for i, scenario in enumerate(sorted(guard.SCENARIOS)): + cases.append({'role': sorted(guard.ROLES)[i % len(guard.ROLES)], 'scenario': scenario, 'kind': ['counts', 'dashboard', 'work-records'][i % 3], 'authorityDigest': 'a' * 64, 'queryDigest': 'a' * 64, 'authorityAllowed': scenario not in ['cross-supplier', 'cross-organization', 'permission-revoked'], 'queryAllowed': scenario not in ['cross-supplier', 'cross-organization', 'permission-revoked'], 'productionHeadKey': 'p', 'workforceHeadKey': 'w', 'authorityScopeDigest': 'scope', 'queryScopeDigest': 'scope'}) + e = {'replicaFactEpoch': 3, 'environment': 'shadow-live', 'queryDatabase': 'drobodata_fanyu_query_shadow_20261114', 'sourceCommit': 'candidate', 'namespaceUID': 'shadow-uid', 'observedAt': '2026-11-16T12:00:00Z', 'cohortDigest': hashlib.sha256(json.dumps(snap['global']['heads'], sort_keys=True, separators=(',', ':')).encode()).hexdigest(), 'cases': cases} + return (e, snap) + + def test_shadow_keeps_budget_safety_and_nonce(self): + args = guard.bounded_shadow_args(self.plan(), 'nonce') + self.assertIn('--safety-run=same-run', args) + self.assertIn('--max-source-records=15000000', args) + self.assertIn('--resume-pause=nonce', args) + p = self.plan() + p['shadowPolicy']['maxRecords'] = 0 + with self.assertRaises(RuntimeError): + guard.bounded_shadow_args(p) + + def test_approved_throughput_stages_are_not_capped_at_100(self): + for batch in (100, 150, 200): + p = self.plan() + p['shadowPolicy']['batchSize'] = batch + p['shadowPolicy']['recordsPerSecond'] = batch + self.assertIn('--batch-size=' + str(batch), guard.bounded_shadow_args(p)) + p['shadowPolicy']['batchSize'] = -1 + with self.assertRaises(RuntimeError): + guard.bounded_shadow_args(p) + + def test_production_writes_need_explicit_scope(self): + with self.assertRaises(RuntimeError): + guard.approved_plan(SimpleNamespace(command='shadow-sync'), None) + self.assertIsNone(guard.approved_plan(SimpleNamespace(command='preflight'), None)) + + def test_real_comparison_requires_results_permissions_scope_and_coverage(self): + e, s = self.evidence() + now = dt.datetime(2026, 11, 16, 12, 1, tzinfo=dt.timezone.utc) + guard.require_result_comparison(e, s, self.plan(), now) + for key, value in [('queryDigest', 'b' * 64), ('queryAllowed', True), ('queryScopeDigest', 'other')]: + changed = json.loads(json.dumps(e)) + changed['cases'][0][key] = value + with self.assertRaises(RuntimeError): + guard.require_result_comparison(changed, s, self.plan(), now) + for key, value in [('cases', e['cases'][:2]), ('environment', 'synthetic'), ('namespaceUID', 'other'), ('cohortDigest', 'other')]: + changed = dict(e) + changed[key] = value + with self.assertRaises(RuntimeError): + guard.require_result_comparison(changed, s, self.plan(), now) + + def test_stale_evidence_not_repackaged_as_fresh(self): + e, s = self.evidence() + with self.assertRaises(RuntimeError): + guard.require_result_comparison(e, s, self.plan(), dt.datetime(2026, 11, 16, 13, tzinfo=dt.timezone.utc)) + + def test_cutover_cannot_be_implied_by_shadow_approval(self): + p = self.plan() + p.update(operations=['cutover-api-flags'], context='ctx', serverSHA256='a' * 64, allowProductionCutover=False) + with tempfile.TemporaryDirectory() as d: + f = Path(d) / 'plan.json' + raw = json.dumps(p).encode() + f.write_bytes(raw) + a = SimpleNamespace(command='cutover-api-flags', approved_plan=str(f), confirm_plan=hashlib.sha256(raw).hexdigest(), approval_ref='explicit-shadow-only') + with self.assertRaisesRegex(RuntimeError, 'production entry'): + guard.approved_plan(a, None) +if __name__ == '__main__': + unittest.main() diff --git a/scripts/local/test_query_replica_runner.py b/scripts/local/test_query_replica_runner.py index a366ccce..423fda2c 100644 --- a/scripts/local/test_query_replica_runner.py +++ b/scripts/local/test_query_replica_runner.py @@ -61,5 +61,29 @@ class ReplicaRunnerBatchSafety(unittest.TestCase): change.assert_not_called() + def test_restart_mount_order_is_irrelevant_but_mount_identity_is_not(self): + import json + from copy import deepcopy + before = {'Id': 'owned', 'Name': '/fanyu-query-replica-local-query-1', + 'HostConfig': {'Memory': 1024 ** 3, 'NanoCpus': 1_000_000_000}, + 'Config': {'Labels': {'com.docker.compose.project': 'fanyu-query-replica-local'}}, + 'Mounts': [{'Destination': '/data/db', 'Name': 'data'}, + {'Destination': '/data/configdb', 'Name': 'config'}]} + for changed in (False, True): + after = deepcopy(before) + after['Mounts'].reverse() + if changed: + after['Mounts'][0]['Name'] = 'different-volume' + with patch.object(runner.subprocess, 'check_output', side_effect=[json.dumps([before]), json.dumps([after])]), \ + patch.object(runner, 'query_probe', return_value=(0, '')), \ + patch.object(runner, 'compose'): + if changed: + with self.assertRaisesRegex(RuntimeError, 'identity'): + runner.restart_query_fixture({}, 'synthetic-probe', 7) + else: + proof = runner.restart_query_fixture({}, 'synthetic-probe', 7) + self.assertTrue(proof['sameVolume'] and proof['majorityJournalRecovered']) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_query_replica.py b/scripts/test_query_replica.py index edffd8fb..2541f8a3 100644 --- a/scripts/test_query_replica.py +++ b/scripts/test_query_replica.py @@ -179,7 +179,9 @@ def restart_query_fixture(env, probe, cursor): raise RuntimeError('restart canary persistence failed') compose(env, 'restart', '-t', '30', 'query') after = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0] - if (after['Id'] != before['Id'] or after['Mounts'] != before['Mounts'] + if (after['Id'] != before['Id'] + or sorted(after['Mounts'], key=lambda m: json.dumps(m, sort_keys=True)) + != sorted(before['Mounts'], key=lambda m: json.dumps(m, sort_keys=True)) or after['HostConfig']['Memory'] != before['HostConfig']['Memory'] or after['HostConfig']['NanoCpus'] != before['HostConfig']['NanoCpus']): raise RuntimeError('query restart changed volume or resource identity') diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index d85fdec8..b0beae7f 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1,13 +1,30 @@ { "version": 1, "scopes": [ + { + "name": "query-cutover-safety", + "paths": [ + "^scripts/local/(evening-query-cutover|query_cutover_guard|test_evening_query_cutover|test_query_cutover_guard)\\.py$", + "^docs/operations/account-role-pending-indexes\\.json$" + ], + "frontend": [], + "go": {}, + "task": [ + { + "argv": ["python3", "-m", "unittest", "discover", "-s", "scripts/local", "-p", "test_*cutover*.py"], + "tests": true + } + ] + }, { "name": "account-role-handoff", "paths": [ "^internal/domain/account_role_work\\.go$", "^internal/api/account_(hierarchy|role_handoff_test)\\.go$", "^internal/store/account_role_(change|handoff_mongo_test)\\.go$", - "^internal/api/s07_saved_slice_recovery_test\\.go$" + "^internal/api/s07_saved_slice_recovery_test\\.go$", + "^internal/api/account_role_guard_bounded_test\\.go$", + "^internal/store/account_role_pending_local_test\\.go$" ], "frontend": [], "go": { @@ -35,7 +52,25 @@ "-timeout=90s", "-v" ], - "env": { "ACCOUNT_ROLE_LOCAL_MONGO": "1" }, + "env": { + "ACCOUNT_ROLE_LOCAL_MONGO": "1" + }, + "tests": true + }, + { + "argv": [ + "go", + "test", + "./internal/store", + "-run", + "^TestAccountRolePendingBoundedParityLocal$", + "-count=1", + "-timeout=90s", + "-v" + ], + "env": { + "QUERY_REPLICA_LOCAL_TEST": "1" + }, "tests": true } ] @@ -375,7 +410,10 @@ "-timeout=3m", "-v" ], - "env": { "POOL_COUNT_LOCAL_MONGO": "1", "POOL_LOCAL_MONGO_URI": "mongodb://127.0.0.1:27018" }, + "env": { + "POOL_COUNT_LOCAL_MONGO": "1", + "POOL_LOCAL_MONGO_URI": "mongodb://127.0.0.1:27018" + }, "tests": true }, { @@ -555,7 +593,10 @@ "argv": ["npm", "run", "test:browser:dashboard"], "tests": true }, - { "argv": ["node", "tests/browser/dashboard-publication.mjs"], "tests": true }, + { + "argv": ["node", "tests/browser/dashboard-publication.mjs"], + "tests": true + }, { "argv": ["python3", "scripts/test_dashboard_stability.py"] }, @@ -647,7 +688,9 @@ "task": [ { "argv": ["go", "test", "./internal/store", "-run", "^TestPoolSearchMetadataWriteCost$", "-count=1", "-v"], - "env": { "SUBMISSION_LOCK_COST_TEST": "1" }, + "env": { + "SUBMISSION_LOCK_COST_TEST": "1" + }, "tests": true }, { @@ -1091,7 +1134,8 @@ "^scripts/local/query-replica\\.query25-local\\.compose\\.yml$", "^scripts/local/query-replica\\.compose\\.yml$", "^Dockerfile\\.query-replica$", - "^docs/(adr/0014-.*|operations/query-replica\\.md|operations/query-build-lifecycle\\.md)$" + "^docs/(adr/0014-.*|operations/query-replica\\.md|operations/query-build-lifecycle\\.md)$", + "^internal/store/query_model_maintenance\\.go$" ], "frontend": [], "go": { @@ -1106,7 +1150,7 @@ "^TestQueryBuildPacing" ], "./cmd/query-cdc-probe": ["^TestClassification"], - "./cmd/query-replica": ["^TestPassDelay", "^TestPacing"], + "./cmd/query-replica": ["^TestPassDelay", "^TestPacing", "^TestDaemonRetryDoesNotHideHardFailures$"], "./cmd/query-lab-bridge": ["^TestIsolatedBridge"], "./internal/testsupport": ["^TestQueryReplicaTarget"] },