diff --git a/docs/adr/0010-source-backed-slice-revisions.md b/docs/adr/0010-source-backed-slice-revisions.md index f6e9fb28..b662ddc9 100644 --- a/docs/adr/0010-source-backed-slice-revisions.md +++ b/docs/adr/0010-source-backed-slice-revisions.md @@ -62,3 +62,13 @@ B06/part0002、B14/part0001、B18/part0003 的本地样本均为连续 30fps,p ## 2026-11-09:送检失败的数据保留 已有返工或动作内容不得通过普通切片生成替换为空工作区。失败送检的持久输入在作业替换前必须留存;只有已发布对应正式标注版本并进入质检,才允许按原保留策略清理旧修订。失败快照写入不成功时停止替换,不能继续清理。旧客户端排队作业同样执行发布前保护。契约与回退风险见 [送检恢复保护](../operations/submission-recovery-guards.md)。 + +## 2026-11-11:隔离异常任务、旧任务限界过渡 + +用户确认优先保障正常开工,将看板和查询优化与全面强制补验分开发布。默认仍为严格完整源校验;仅 API 与切片处理器显式设置相同 `ANNOTATION_LEGACY_SOURCE_CUTOFF=2026-11-12T00:00:00Z` 时启用本次过渡。该 UTC 边界对应洛杉矶 11 月 11 日 16:00,生产配置须随具体发布计划审批。 + +过渡仅适用于截止时间前创建、状态 completed、没有完整源指纹且未标记完整重校验的旧时间轴,以及没有源索引物化/作业的原物理流程。任何已知失败、排队或运行的关联源作业、跨组织/项目/Run 记录、已知指纹不匹配均继续阻断。新任务、显式完整补验和源索引流程维持严格校验;读取不触发补验。 + +列表仍返回服务端可领取判定;空 sourceValidationStatus 仅表示没有准入阻断,不是完整源证书。任务详情对过渡记录返回 unverified、required=false,不写造假的 fingerprint,不将后台作业改成成功。异常任务保留工作区、草稿、归属及审核历史,不清空、不自动回池;管理员保留异常查询入口。 + +后台补验/源修复按独立固定清单及生产数据授权执行;补验状态不通过改写旧时间轴成功证据来隐藏失败。实际补验期间进入严格校验的任务可能暂时阻断,须有明确窗口或使用独立受审查的后台审计方案,不能声称无中断。收紧回严格默认前,先核对旧任务和公共池的真实证书准备度;移除配置会重新阻断尚未完成补验的旧任务,不能盲目回退。 diff --git a/docs/architecture/source-validation-admission.md b/docs/architecture/source-validation-admission.md index fbfa3ed1..d269f0c9 100644 --- a/docs/architecture/source-validation-admission.md +++ b/docs/architecture/source-validation-admission.md @@ -43,3 +43,13 @@ 本地 1,200 条任务、每条 64 KiB subject 的 Mongo 夹具,原始 subject 共 75 MiB,候选 JSON 为约 0.49 MiB。新增状态关联比原来不做准入判断的计数更贵;实际对照数值见[验证记录](../verification/source-validation-admission.md)。该测试不能替代 120 人完整作业验收。 接口新增字段可由旧客户端忽略。恢复作业需要理解 `revalidateSource` 的新版校验器;旧校验器仅完成时间轴而未提供源指纹证明时,新 API 保持未核验,不据此放行。发布须协调 API、校验器及接入算子版本;未执行生产回填、生产源修复或上线。 + +## 2026-11-11:隔离异常任务、旧任务限界过渡 + +用户确认优先保障正常开工,将看板和查询优化与全面强制补验分开发布。默认仍为严格完整源校验;仅 API 与切片处理器显式设置相同 `ANNOTATION_LEGACY_SOURCE_CUTOFF=2026-11-12T00:00:00Z` 时启用本次过渡。该 UTC 边界对应洛杉矶 11 月 11 日 16:00,生产配置须随具体发布计划审批。 + +过渡仅适用于截止时间前创建、状态 completed、没有完整源指纹且未标记完整重校验的旧时间轴,以及没有源索引物化/作业的原物理流程。任何已知失败、排队或运行的关联源作业、跨组织/项目/Run 记录、已知指纹不匹配均继续阻断。新任务、显式完整补验和源索引流程维持严格校验;读取不触发补验。 + +列表仍返回服务端可领取判定;空 sourceValidationStatus 仅表示没有准入阻断,不是完整源证书。任务详情对过渡记录返回 unverified、required=false,不写造假的 fingerprint,不将后台作业改成成功。异常任务保留工作区、草稿、归属及审核历史,不清空、不自动回池;管理员保留异常查询入口。 + +后台补验/源修复按独立固定清单及生产数据授权执行;补验状态不通过改写旧时间轴成功证据来隐藏失败。实际补验期间进入严格校验的任务可能暂时阻断,须有明确窗口或使用独立受审查的后台审计方案,不能声称无中断。收紧回严格默认前,先核对旧任务和公共池的真实证书准备度;移除配置会重新阻断尚未完成补验的旧任务,不能盲目回退。 diff --git a/docs/release/README.md b/docs/release/README.md index 4742ee4a..5e157035 100644 --- a/docs/release/README.md +++ b/docs/release/README.md @@ -241,3 +241,7 @@ Mongo 27018 合成隔离库,包含 24000 条/来源规模、历史兼容、故 关键测试未执行、跳过或失败均阻断,不能以无 Mongo 时的 skip 结果发布。 上线时还需审查三个统计索引的创建授权及可选诊断字段的混合版本行为;验收同时检查真实看板 200 与快照持续更新,不能只看进程健康。接口、检查器和边界见[看板持续运行说明](../dashboard-stability.zh-CN.md)。 + +### 旧任务限界过渡配置 + +`ANNOTATION_LEGACY_SOURCE_CUTOFF` 默认未设置,保持严格完整源准入。经 2026-11-11 用户明确授权的过渡配置必须在 API/切片处理器上精确一致,发布快照记录该非秘密值,格式或组件不一致阻断。配置变更仍不属于 release.py 的镜像更新范围,须单列实际部署 UID、原值、新值、顺序和回退影响并批准;不能通过修改工具扩大自动发布权限。业务范围见源校验 ADR。此过渡发布保留现用 Worker 与校验器,删除人员后台接管开关继续关闭,不启动全量补验。 diff --git a/internal/api/annotation_legacy_transition_test.go b/internal/api/annotation_legacy_transition_test.go new file mode 100644 index 00000000..0ee0039d --- /dev/null +++ b/internal/api/annotation_legacy_transition_test.go @@ -0,0 +1,30 @@ +package api + +import ( + "droboticflow/internal/domain" + "testing" +) + +func TestLegacySourceTransitionStatusIsNotFullProof(t *testing.T) { + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "2026-11-12T00:00:00Z") + f := newCombinedDraftFixture(t) + item := f.item + job := domain.AnnotationIndexJob{ID: f.task.ID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, RunID: f.task.RunID, Status: "completed", CreatedAt: "2026-11-11T00:00:00Z"} + wrapped := &sourceProofReadStore{MemoryStore: f.store, timeline: &job} + engine := &server{store: wrapped} + v, e := engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true) + if e != nil { + t.Fatal(e) + } + if v.Status != "unverified" || v.Required || v.blocked() || v.RetryAllowed { + t.Fatalf("legacy admission lied about full proof or blocked old task: %+v", v) + } + job.Status = "failed" + v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true) + if e != nil || v.Status != "failed" || !v.blocked() { + t.Fatalf("known failure not isolated: %+v %v", v, e) + } + if wrapped.taskWrites != 0 { + t.Fatal("read triggered task writes") + } +} diff --git a/internal/api/annotation_source_admission.go b/internal/api/annotation_source_admission.go index 16792422..23b865e1 100644 --- a/internal/api/annotation_source_admission.go +++ b/internal/api/annotation_source_admission.go @@ -127,7 +127,14 @@ func (s *server) annotationSourceValidationForTask(ctx context.Context, task dom // A completed proof for another physical outbox must not appear "valid". switch store.AnnotationSourceAdmissionStatus(task, current, base, source) { case "": - v.Status, v.Code, v.Message = "valid", "", "" + if store.LegacyAnnotationSourceCompatible(task, current, base, source) { + // Eligible under the explicitly approved legacy transition, not + // fully validated. Saving/claiming is permitted without lying + // about a certificate that does not exist. + v.Status, v.Code, v.Message, v.Required = "unverified", "", "", false + } else { + v.Status, v.Code, v.Message = "valid", "", "" + } case "pending": v.Status, v.Code, v.Message = "pending", "source_validation_pending", "源数据校验中" case "failed": diff --git a/internal/store/annotation_claim_read.go b/internal/store/annotation_claim_read.go index b89e70d1..61d780ad 100644 --- a/internal/store/annotation_claim_read.go +++ b/internal/store/annotation_claim_read.go @@ -153,7 +153,7 @@ func (s *MongoStore) EnrichAnnotationSourceStatuses(ctx context.Context, items [ if err != nil { return err } - cur, err = s.db.Collection("annotationIndexJobs").Find(ctx, bson.M{"id": bson.M{"$in": jobIDs}}, options.Find().SetMaxTime(8*time.Second).SetBatchSize(400).SetProjection(bson.M{"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "runId": 1, "status": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1})) + cur, err = s.db.Collection("annotationIndexJobs").Find(ctx, bson.M{"id": bson.M{"$in": jobIDs}}, options.Find().SetMaxTime(8*time.Second).SetBatchSize(400).SetProjection(bson.M{"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "runId": 1, "status": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1})) if err != nil { return err } diff --git a/internal/store/annotation_legacy_transition_test.go b/internal/store/annotation_legacy_transition_test.go new file mode 100644 index 00000000..47cd7324 --- /dev/null +++ b/internal/store/annotation_legacy_transition_test.go @@ -0,0 +1,192 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "encoding/json" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "reflect" + "testing" + "time" +) + +type legacyTransitionCase struct { + name string + change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob) + allowed bool +} + +func legacyTransitionCases() []legacyTransitionCase { + return []legacyTransitionCase{ + {name: "old-completed", allowed: true}, + {name: "known-failed-timeline", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.Status = "failed" + }}, + {name: "pending-timeline", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.Status = "queued" + }}, + {name: "new-job", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.CreatedAt = "2026-11-12T00:00:00Z" + }}, + {name: "missing-date", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.CreatedAt = "" + }}, + {name: "date-only", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.CreatedAt = "2026-11-11" + }}, + {name: "malformed-date", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.CreatedAt = "not-a-date" + }}, + {name: "explicit-full-revalidation", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.RevalidateSource = true + }}, + {name: "indexed-outbox", change: func(i *domain.HumanItemRecord, _ *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + i.ProcessingJob.SourceIndex = true + }}, + {name: "indexed-materialization", change: func(i *domain.HumanItemRecord, _ *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + i.Subject.Payload["materialization"] = map[string]any{"sourceIndex": true} + }}, + {name: "foreign-timeline", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { + j.OrganizationID = "foreign" + }}, + {name: "known-failed-source", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.Status = "failed" + *s = &v + }}, + {name: "pending-source", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.Status = "running" + *s = &v + }}, + {name: "wrong-source-proof", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.SourceFingerprint = "different" + *s = &v + }}, + {name: "current-source-proof", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.SourceFingerprint = "current" + *s = &v + }}, + } +} +func legacyTransitionRecords(n int, c legacyTransitionCase) (domain.AnnotationTaskRecord, domain.HumanItemRecord, domain.AnnotationIndexJob, *domain.AnnotationIndexJob) { + id := fmt.Sprintf("legacy-case-%d", n) + task := domain.AnnotationTaskRecord{ID: id, OrganizationID: "legacy-org", ProjectID: "legacy-project", RunID: id} + item := domain.HumanItemRecord{ID: id, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: id, LegacyTaskID: id, AssignedUserID: "preserve-owner", Version: 3, ProcessingJob: &domain.AnnotationProcessingJob{SourceFingerprint: id + "-current"}, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationWorkspace": map[string]any{"description": "preserve"}}}} + j := domain.AnnotationIndexJob{ID: id, OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, RunID: id, Status: "completed", CreatedAt: "2026-11-11T23:59:59.999999999Z"} + var src *domain.AnnotationIndexJob + if c.change != nil { + c.change(&item, &j, &src) + } + if src != nil { + src.ID = AnnotationSourceJobID(task.OrganizationID, item.ProcessingJob.SourceFingerprint) + if src.SourceFingerprint == "current" { + src.SourceFingerprint = item.ProcessingJob.SourceFingerprint + } + } + return task, item, j, src +} +func TestLegacySourceTransitionFence(t *testing.T) { + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "2026-11-12T00:00:00Z") + for n, c := range legacyTransitionCases() { + t.Run(c.name, func(t *testing.T) { + task, item, j, src := legacyTransitionRecords(n, c) + before, _ := json.Marshal([]any{item, j, src}) + if got := LegacyAnnotationSourceCompatible(task, item, &j, src); got != c.allowed { + t.Fatalf("compatibility=%v want=%v", got, c.allowed) + } + if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != c.allowed { + t.Fatalf("ready=%v want=%v", got, c.allowed) + } + if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != c.allowed { + t.Fatal("projection differs from eligibility") + } + after, _ := json.Marshal([]any{item, j, src}) + if string(before) != string(after) { + t.Fatal("inputs changed") + } + }) + } + for _, bad := range []string{"", "invalid", "2026-11-12T00:00:00+00:00", "2026-11-12T00:00:00.1Z"} { + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", bad) + task, item, j, src := legacyTransitionRecords(0, legacyTransitionCases()[0]) + if AnnotationItemSourceJobsReady(task, item, &j, src) { + t.Fatal("invalid or absent opt-in allowed weak proof") + } + } +} +func TestLegacySourceTransitionMongoEquivalence(t *testing.T) { + if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" { + t.Skip("isolated Mongo opt-in") + } + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "2026-11-12T00:00:00Z") + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(3*time.Second)) + if e != nil { + t.Fatal(e) + } + defer client.Disconnect(context.Background()) + db := client.Database(fmt.Sprintf("legacy_source_transition_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + s := NewMongoStore(db) + items := []domain.HumanItemRecord{} + expected := map[string]bool{} + for n, c := range legacyTransitionCases() { + task, item, j, src := legacyTransitionRecords(n, c) + _, e = db.Collection("annotationTasks").InsertOne(ctx, task) + if e != nil { + t.Fatal(e) + } + _, e = db.Collection("annotationIndexJobs").InsertOne(ctx, j) + if e != nil { + t.Fatal(e) + } + if src != nil { + _, e = db.Collection("annotationIndexJobs").InsertOne(ctx, src) + if e != nil { + t.Fatal(e) + } + } + _, e = db.Collection("humanItems").InsertOne(ctx, item) + if e != nil { + t.Fatal(e) + } + items = append(items, item) + if c.allowed { + expected[item.ID] = true + } + } + cur, e := db.Collection("humanItems").Aggregate(ctx, annotationSourceAdmissionPipeline()) + if e != nil { + t.Fatal(e) + } + rows, e := decodeMany[domain.HumanItemRecord](ctx, cur) + if e != nil { + t.Fatal(e) + } + actual := map[string]bool{} + for _, r := range rows { + actual[r.ID] = true + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("aggregation=%v expected=%v", actual, expected) + } + if e = s.EnrichAnnotationSourceStatuses(ctx, items); e != nil { + t.Fatal(e) + } + for _, i := range items { + if (i.SourceValidationStatus == "") != expected[i.ID] { + t.Fatalf("batch projection differs %s status=%s", i.ID, i.SourceValidationStatus) + } + } +} diff --git a/internal/store/annotation_source_admission.go b/internal/store/annotation_source_admission.go index 2282a3f4..3b56ecf6 100644 --- a/internal/store/annotation_source_admission.go +++ b/internal/store/annotation_source_admission.go @@ -3,6 +3,9 @@ package store import ( "context" "errors" + "os" + "regexp" + "time" "droboticflow/internal/domain" "go.mongodb.org/mongo-driver/bson" @@ -71,7 +74,40 @@ func annotationOutboxProofMatches(item domain.HumanItemRecord, timeline, source return timeline != nil && timeline.SourceFingerprint == fingerprint } +var legacySourceCreatedAt = regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{1,9})?Z$`) + +// LegacySourceAdmissionCutoff is an explicit, temporary deployment boundary. +// Missing/invalid configuration keeps strict full-source admission enabled. +func LegacySourceAdmissionCutoff() (time.Time, bool) { + raw := os.Getenv("ANNOTATION_LEGACY_SOURCE_CUTOFF") + cutoff, err := time.Parse(time.RFC3339, raw) + return cutoff, err == nil && raw == cutoff.UTC().Format(time.RFC3339) && cutoff.Nanosecond() == 0 +} + +// LegacyAnnotationSourceCompatible preserves only the previously admitted +// physical workflow. It never manufactures a source fingerprint/certificate. +func LegacyAnnotationSourceCompatible(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool { + cutoff, enabled := LegacySourceAdmissionCutoff() + if !enabled || task.ID == "" || timeline == nil || timeline.ID != task.ID || timeline.RevalidateSource || timeline.SourceFingerprint != "" || AnnotationSourceCertificateRequired(item) { + return false + } + created, err := time.Parse(time.RFC3339Nano, timeline.CreatedAt) + if err != nil || !legacySourceCreatedAt.MatchString(timeline.CreatedAt) || !created.Before(cutoff) { + return false + } + for _, job := range []*domain.AnnotationIndexJob{timeline, source} { + if job != nil && (job.Status != "completed" || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) { + return false + } + } + // A known current-source proof must match the physical outbox identity. + return source == nil || source.SourceFingerprint != "" && item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint +} + func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool { + if LegacyAnnotationSourceCompatible(task, item, timeline, source) { + return true + } if timeline != nil && timeline.RevalidateSource { if timeline.SourceFingerprint == "" || !AnnotationSourceJobsReady(task, timeline, nil) { return false @@ -204,9 +240,25 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline { outboxProofMatches, bson.M{"$allElementsTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": safeJob}}}}, }} + // Only the API/processor deployment explicitly opting into the frozen + // legacy window may reuse old completed time-axis admission. Failed or + // pending jobs remain excluded; source-index workflows still need proof. + if cutoff, enabled := LegacySourceAdmissionCutoff(); enabled { + created := bson.M{"$convert": bson.M{"input": first("$_sourceTimeline.createdAt"), "to": "date", "onError": nil, "onNull": nil}} + legacy := bson.M{"$and": bson.A{taskPresent, timelinePresent, + bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourceTimeline.createdAt")}, "string"}}, first("$_sourceTimeline.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}}, + bson.M{"$not": bson.A{revalidated}}, bson.M{"$not": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint"))}}, + bson.M{"$ne": bson.A{created, nil}}, bson.M{"$lt": bson.A{created, cutoff}}, + bson.M{"$not": bson.A{needsCertificate}}, sourceVersionMatches, + bson.M{"$allElementsTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, matchingScope}}}}}}, + bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}}, + }} + ready = bson.M{"$or": bson.A{ready, legacy}} + missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{legacy}}}} + } active := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$in": bson.A{"$$job.status", bson.A{"queued", "running"}}}}}}} failed := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$$job.status", "failed"}}, bson.M{"$not": bson.A{matchingScope}}}}}}}} - jobProjection := bson.M{"status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1} + jobProjection := bson.M{"status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1} lookup := func(collection, local, as string, projection bson.M) bson.D { return bson.D{{Key: "$lookup", Value: bson.M{"from": collection, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$project": projection}}, "as": as}}} } diff --git a/scripts/release.py b/scripts/release.py index 5248acf4..d3350be9 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -21,7 +21,7 @@ SAFE_ENV = ('MONGO_DB', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH', 'ANNOTATION_PROCESSOR_MODE', 'ANNOTATION_PROCESSOR_CONCURRENCY', 'ANNOTATION_PROCESSOR_ROLE', 'ANNOTATION_SOURCE_INDEX', 'ANNOTATION_INDEX_CONCURRENCY', 'DROBOTICFLOW_DATABASE_BOOTSTRAP', 'DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE', - 'ANNOTATION_INDEX_BACKFILL', 'ANNOTATION_WORKSPACE_CONNECTION_ID', + 'ANNOTATION_INDEX_BACKFILL', 'ANNOTATION_LEGACY_SOURCE_CUTOFF', 'ANNOTATION_WORKSPACE_CONNECTION_ID', 'ANNOTATION_WORKSPACE_BUCKET', 'WORKSPACE_MOUNT_PATH') @@ -236,6 +236,18 @@ def inspect_deployment(deployment, container_name, configs, p): 'ready': ready, 'env': {k: e.get(k) for k in SAFE_ENV}} +def validate_legacy_source_cutoffs(envs): + cutoffs = {e.get('ANNOTATION_LEGACY_SOURCE_CUTOFF') for e in envs} + require(len(cutoffs) == 1, 'API/processor legacy source admission cutoffs differ.') + cutoff = next(iter(cutoffs)) + if cutoff: + require(bool(re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', cutoff)), 'Invalid legacy source admission cutoff.') + try: + dt.datetime.strptime(cutoff, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + raise Blocked('Invalid legacy source admission cutoff.') + + def snapshot(r): t, p = r['target'], policy() view = json.loads(kubectl(r, ['config', 'view', '--minify', '-o', 'json'])) @@ -253,6 +265,7 @@ def snapshot(r): deps = {n: inspect_deployment(d, p['deployments'][n], configs, p) for n, d in deployments.items()} envs = [d['env'] for d in deps.values()] require(len({e['ANNOTATION_WORKSPACE_CONNECTION_ID'] for e in envs}) == 1, 'API/processor storage connections differ.') + validate_legacy_source_cutoffs(envs) pvc = get(r, 'pvc', p['workspacePVC']) require(pvc.get('status', {}).get('phase') == 'Bound', 'Workspace PVC is not Bound.') svc = get(r, 'service', 'droboticflow-api-public-eip') diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 1c7dec41..8359dd6e 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -176,7 +176,7 @@ "name": "source-and-buffer", "paths": [ "^src/(pages/AnnotationOperations\\.tsx|services/api/annotation-operations\\.ts)$", - "^internal/(api|store)/annotation_(source|work_buffer|claim_read|claim_visit|random_claim)", + "^internal/(api|store)/annotation_(source|legacy_transition|work_buffer|claim_read|claim_visit|random_claim)", "^internal/store/human_claim", "^internal/store/admin_source_exceptions_test\\.go$", "^internal/store/pool_count_performance_test\\.go$", @@ -191,9 +191,10 @@ "^TestSourceValidation", "^TestPendingSourceValidation", "^TestWorkBufferRetains", - "^TestRandomClaim" + "^TestRandomClaim", + "^TestLegacySourceTransition" ], - "./internal/store": ["^TestSourceAdmissionMemoryClaimFence$"] + "./internal/store": ["^TestSourceAdmissionMemoryClaimFence$", "^TestLegacySourceTransitionFence$"] }, "task": [ { @@ -202,7 +203,7 @@ "test", "./internal/store", "-run", - "^TestSourceAdmission(Mongo(ClaimFence|BatchMatchesLegacyPipeline|ClaimReadPlan|LargePool)|CurrentOutboxProofMongo|ExistingTaskUpdateDoesNotEnqueueMongo)$|^TestAdminSourceExceptionsMongo$", + "^TestSourceAdmission(Mongo(ClaimFence|BatchMatchesLegacyPipeline|ClaimReadPlan|LargePool)|CurrentOutboxProofMongo|ExistingTaskUpdateDoesNotEnqueueMongo)$|^TestAdminSourceExceptionsMongo$|^TestLegacySourceTransitionMongoEquivalence$", "-count=1", "-timeout=3m", "-v" @@ -239,7 +240,9 @@ "-count=1", "-v" ], - "env": { "SOURCE_ADMISSION_LOCAL_MONGO": "1" }, + "env": { + "SOURCE_ADMISSION_LOCAL_MONGO": "1" + }, "tests": true }, { @@ -824,6 +827,18 @@ } ], "warnings": ["索引建设、历史返工元数据补处理和开关启用均须单独批准;本地性能对照不代表生产容量通过。"] + }, + { + "name": "release-config-guards", + "paths": ["^scripts/release\\.py$", "^tests/release/test_release\\.py$"], + "frontend": [], + "go": {}, + "task": [ + { + "argv": ["python3", "-m", "unittest", "discover", "-s", "tests/release", "-p", "test_release.py"], + "tests": true + } + ] } ] } diff --git a/tests/release/test_release.py b/tests/release/test_release.py index e7834731..4b652d66 100644 --- a/tests/release/test_release.py +++ b/tests/release/test_release.py @@ -496,5 +496,20 @@ class EntrypointTests(unittest.TestCase): self.assertFalse(called.exists()) +class LegacySourceCutoffTests(unittest.TestCase): + def test_strict_default_and_approved_equal_cutoff(self): + m.validate_legacy_source_cutoffs([{}, {}]) + v = {'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-11-12T00:00:00Z'} + m.validate_legacy_source_cutoffs([v, v]) + + def test_mismatched_and_invalid_cutoffs_block(self): + for values in [({}, {'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-11-12T00:00:00Z'}), + ({'ANNOTATION_LEGACY_SOURCE_CUTOFF': 'bad'},)*2, + ({'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-02-30T00:00:00Z'},)*2, + ({'ANNOTATION_LEGACY_SOURCE_CUTOFF': '2026-11-12T00:00:00.1Z'},)*2]: + with self.subTest(values=values), self.assertRaises(m.Blocked): + m.validate_legacy_source_cutoffs(values) + + if __name__ == '__main__': unittest.main()