diff --git a/docs/architecture/source-validation-admission.md b/docs/architecture/source-validation-admission.md index d269f0c9..e0282a42 100644 --- a/docs/architecture/source-validation-admission.md +++ b/docs/architecture/source-validation-admission.md @@ -53,3 +53,5 @@ 列表仍返回服务端可领取判定;空 sourceValidationStatus 仅表示没有准入阻断,不是完整源证书。任务详情对过渡记录返回 unverified、required=false,不写造假的 fingerprint,不将后台作业改成成功。异常任务保留工作区、草稿、归属及审核历史,不清空、不自动回池;管理员保留异常查询入口。 后台补验/源修复按独立固定清单及生产数据授权执行;补验状态不通过改写旧时间轴成功证据来隐藏失败。实际补验期间进入严格校验的任务可能暂时阻断,须有明确窗口或使用独立受审查的后台审计方案,不能声称无中断。收紧回严格默认前,先核对旧任务和公共池的真实证书准备度;移除配置会重新阻断尚未完成补验的旧任务,不能盲目回退。 + +旧时间轴或源作业的空指纹不等于已知非空指纹冲突:两条旧 completed 记录都在截止时间前、均非完整重校验、且没有源索引时,可使用同一过渡规则,仍呈现未完整校验。反之,已有匹配当前 outbox、同组织/项目/Run 的完整源作业证明时,不要求旧时间轴重复持有同一指纹;这是真实完整证明,不是源索引流程的豁免。任何失败、活动作业、非空指纹冲突、显式重校验尚未完成仍阻断。 diff --git a/internal/api/annotation_legacy_transition_test.go b/internal/api/annotation_legacy_transition_test.go index 0ee0039d..b47e3337 100644 --- a/internal/api/annotation_legacy_transition_test.go +++ b/internal/api/annotation_legacy_transition_test.go @@ -2,6 +2,7 @@ package api import ( "droboticflow/internal/domain" + "droboticflow/internal/store" "testing" ) @@ -28,3 +29,33 @@ func TestLegacySourceTransitionStatusIsNotFullProof(t *testing.T) { t.Fatal("read triggered task writes") } } + +func TestLegacySourceMetadataGapKeepsUnknownSeparateFromMismatch(t *testing.T) { + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "2026-11-12T00:00:00Z") + f := newCombinedDraftFixture(t) + item := f.item + item.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: "current"} + base := domain.AnnotationIndexJob{ID: f.task.ID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, RunID: f.task.RunID, Status: "completed", CreatedAt: "2026-10-01T00:00:00Z"} + src := base + src.ID = store.AnnotationSourceJobID(f.task.OrganizationID, "current") + wrapped := &sourceProofReadStore{MemoryStore: f.store, timeline: &base, source: &src} + engine := &server{store: wrapped} + v, e := engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true) + if e != nil || v.Status != "unverified" || v.Required || v.blocked() { + t.Fatalf("old metadata-only gap blocked or reported valid: %+v %v", v, e) + } + src.SourceFingerprint = "different" + v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true) + if e != nil || !v.blocked() { + t.Fatalf("real mismatch admitted: %+v %v", v, e) + } + src.SourceFingerprint = "current" + item.ProcessingJob.SourceIndex = true + v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true) + if e != nil || v.Status != "valid" || v.blocked() || !v.Required { + t.Fatalf("matching full source proof ignored: %+v %v", v, e) + } + if wrapped.taskWrites != 0 { + t.Fatal("status read wrote metadata") + } +} diff --git a/internal/store/annotation_legacy_transition_test.go b/internal/store/annotation_legacy_transition_test.go index 47cd7324..6d121016 100644 --- a/internal/store/annotation_legacy_transition_test.go +++ b/internal/store/annotation_legacy_transition_test.go @@ -14,9 +14,10 @@ import ( ) type legacyTransitionCase struct { - name string - change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob) - allowed bool + name string + change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob) + allowed bool + fullProof bool } func legacyTransitionCases() []legacyTransitionCase { @@ -52,6 +53,23 @@ func legacyTransitionCases() []legacyTransitionCase { {name: "foreign-timeline", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) { j.OrganizationID = "foreign" }}, + {name: "old-source-missing-metadata", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + *src = &v + }}, + {name: "new-source-missing-metadata", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.CreatedAt = "2026-11-12T00:00:00Z" + *src = &v + }}, + {name: "revalidated-source-missing-metadata", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) { + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.RevalidateSource = true + *src = &v + }}, {name: "known-failed-source", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { v := *j v.ID = AnnotationSourceJobID(j.OrganizationID, "current") @@ -70,6 +88,13 @@ func legacyTransitionCases() []legacyTransitionCase { v.SourceFingerprint = "different" *s = &v }}, + {name: "indexed-existing-complete-proof", fullProof: true, change: func(i *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { + i.ProcessingJob.SourceIndex = true + v := *j + v.ID = AnnotationSourceJobID(j.OrganizationID, "current") + v.SourceFingerprint = "current" + *s = &v + }}, {name: "current-source-proof", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) { v := *j v.ID = AnnotationSourceJobID(j.OrganizationID, "current") @@ -104,10 +129,10 @@ func TestLegacySourceTransitionFence(t *testing.T) { if got := LegacyAnnotationSourceCompatible(task, item, &j, src); got != c.allowed { t.Fatalf("compatibility=%v want=%v", got, c.allowed) } - if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != c.allowed { + if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != (c.allowed || c.fullProof) { t.Fatalf("ready=%v want=%v", got, c.allowed) } - if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != c.allowed { + if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != (c.allowed || c.fullProof) { t.Fatal("projection differs from eligibility") } after, _ := json.Marshal([]any{item, j, src}) @@ -162,7 +187,7 @@ func TestLegacySourceTransitionMongoEquivalence(t *testing.T) { t.Fatal(e) } items = append(items, item) - if c.allowed { + if c.allowed || c.fullProof { expected[item.ID] = true } } @@ -190,3 +215,26 @@ func TestLegacySourceTransitionMongoEquivalence(t *testing.T) { } } } + +func TestLegacyTimelineAcceptsOnlyMatchingCompleteCurrentSource(t *testing.T) { + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "") + task, item, j, src := legacyTransitionRecords(0, legacyTransitionCases()[0]) + item.ProcessingJob.SourceIndex = true + full := j + full.ID = AnnotationSourceJobID(task.OrganizationID, item.ProcessingJob.SourceFingerprint) + full.SourceFingerprint = item.ProcessingJob.SourceFingerprint + src = &full + if !AnnotationItemSourceJobsReady(task, item, &j, src) { + t.Fatal("complete current source proof ignored because legacy timeline lacks metadata") + } + for _, change := range []func(){func() { src.SourceFingerprint = "different" }, func() { src.Status = "failed" }, func() { src.Status = "running" }, func() { j.RevalidateSource = true }, func() { src.RunID = "foreign" }} { + j.RevalidateSource = false + src.SourceFingerprint = item.ProcessingJob.SourceFingerprint + src.Status = "completed" + src.RunID = task.RunID + change() + if AnnotationItemSourceJobsReady(task, item, &j, src) { + t.Fatal("invalid or mismatched proof admitted") + } + } +} diff --git a/internal/store/annotation_source_admission.go b/internal/store/annotation_source_admission.go index 3b56ecf6..076e4f7a 100644 --- a/internal/store/annotation_source_admission.go +++ b/internal/store/annotation_source_admission.go @@ -22,11 +22,18 @@ func AnnotationSourceValidationRequired(_ domain.AnnotationTaskRecord) bool { } func AnnotationSourceJobsReady(task domain.AnnotationTaskRecord, timeline, source *domain.AnnotationIndexJob) bool { - if timeline == nil || timeline.SourceFingerprint == "" { + if timeline == nil { return false } - for _, job := range []*domain.AnnotationIndexJob{timeline, source} { - if job != nil && (job.Status != "completed" || job.SourceFingerprint == "" || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) { + // A complete, current source job is full proof even when the historical + // time-axis job predates the fingerprint field. Explicit revalidation is + // never satisfied by a half-published timeline. + historicalWithSourceProof := !timeline.RevalidateSource && source != nil && source.SourceFingerprint != "" + if timeline.SourceFingerprint == "" && !historicalWithSourceProof { + return false + } + for index, job := range []*domain.AnnotationIndexJob{timeline, source} { + if job != nil && (job.Status != "completed" || (job.SourceFingerprint == "" && !(index == 0 && historicalWithSourceProof)) || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) { return false } } @@ -101,7 +108,16 @@ func LegacyAnnotationSourceCompatible(task domain.AnnotationTaskRecord, item dom } } // A known current-source proof must match the physical outbox identity. - return source == nil || source.SourceFingerprint != "" && item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint + if source == nil { + return true + } + if source.SourceFingerprint != "" { + return item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint + } + // Old completed source-job metadata can lack a fingerprint too. This is + // unknown evidence, not a known mismatch; bound both old records to cutoff. + sourceCreated, err := time.Parse(time.RFC3339Nano, source.CreatedAt) + return !source.RevalidateSource && err == nil && legacySourceCreatedAt.MatchString(source.CreatedAt) && sourceCreated.Before(cutoff) } func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool { @@ -212,7 +228,8 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline { needsCertificate := bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$processingJob.sourceIndex", true}}, bson.M{"$eq": bson.A{"$subject.payload.materialization.sourceIndex", true}}}} jobs := bson.M{"$concatArrays": bson.A{"$_sourceTimeline", "$_sourcePrepared"}} matchingScope := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.organizationId", "$organizationId"}}, bson.M{"$eq": bson.A{"$$job.projectId", "$projectId"}}, bson.M{"$eq": bson.A{"$$job.runId", "$runId"}}}} - safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, nonempty("$$job.sourceFingerprint"), matchingScope}} + historicalSourceProof := bson.M{"$and": bson.A{bson.M{"$not": bson.A{revalidated}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}} + safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, bson.M{"$or": bson.A{nonempty("$$job.sourceFingerprint"), bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.id", "$legacyTaskId"}}, historicalSourceProof}}}}, matchingScope}} timelinePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTimeline"}, 0}} taskPresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTask"}, 0}} certificatePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourcePrepared"}, 0}} @@ -234,7 +251,7 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline { ready := bson.M{"$and": bson.A{ taskPresent, timelinePresent, - nonempty(first("$_sourceTimeline.sourceFingerprint")), + bson.M{"$or": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint")), historicalSourceProof}}, certificateSatisfied, sourceVersionMatches, outboxProofMatches, @@ -245,20 +262,24 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline { // pending jobs remain excluded; source-index workflows still need proof. if cutoff, enabled := LegacySourceAdmissionCutoff(); enabled { created := bson.M{"$convert": bson.M{"input": first("$_sourceTimeline.createdAt"), "to": "date", "onError": nil, "onNull": nil}} + sourceCreated := bson.M{"$convert": bson.M{"input": first("$_sourcePrepared.createdAt"), "to": "date", "onError": nil, "onNull": nil}} + sourceOldDate := bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourcePrepared.createdAt")}, "string"}}, first("$_sourcePrepared.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}} + weakOldSource := bson.M{"$and": bson.A{bson.M{"$not": bson.A{nonempty(first("$_sourcePrepared.sourceFingerprint"))}}, bson.M{"$ne": bson.A{first("$_sourcePrepared.revalidateSource"), true}}, sourceOldDate, bson.M{"$ne": bson.A{sourceCreated, nil}}, bson.M{"$lt": bson.A{sourceCreated, cutoff}}}} + legacySourceMatches := bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, bson.M{"$and": bson.A{nonempty(first("$_sourcePrepared.sourceFingerprint")), sourceVersionMatches}}, weakOldSource}} legacy := bson.M{"$and": bson.A{taskPresent, timelinePresent, bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourceTimeline.createdAt")}, "string"}}, first("$_sourceTimeline.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}}, bson.M{"$not": bson.A{revalidated}}, bson.M{"$not": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint"))}}, bson.M{"$ne": bson.A{created, nil}}, bson.M{"$lt": bson.A{created, cutoff}}, - bson.M{"$not": bson.A{needsCertificate}}, sourceVersionMatches, + bson.M{"$not": bson.A{needsCertificate}}, legacySourceMatches, bson.M{"$allElementsTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, matchingScope}}}}}}, - bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}}, }} ready = bson.M{"$or": bson.A{ready, legacy}} missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{legacy}}}} } + missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{ready}}}} active := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$in": bson.A{"$$job.status", bson.A{"queued", "running"}}}}}}} failed := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$$job.status", "failed"}}, bson.M{"$not": bson.A{matchingScope}}}}}}}} - jobProjection := bson.M{"status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1} + jobProjection := bson.M{"id": 1, "status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1} lookup := func(collection, local, as string, projection bson.M) bson.D { return bson.D{{Key: "$lookup", Value: bson.M{"from": collection, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$project": projection}}, "as": as}}} } diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 8359dd6e..7a92c6bf 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -194,7 +194,11 @@ "^TestRandomClaim", "^TestLegacySourceTransition" ], - "./internal/store": ["^TestSourceAdmissionMemoryClaimFence$", "^TestLegacySourceTransitionFence$"] + "./internal/store": [ + "^TestSourceAdmissionMemoryClaimFence$", + "^TestLegacySourceTransitionFence$", + "^TestLegacyTimelineAcceptsOnlyMatchingCompleteCurrentSource$" + ] }, "task": [ {