🛡️ Distinguish historical proof metadata gaps from source conflicts

This commit is contained in:
Codex 2026-11-11 17:35:58 -08:00
parent a2f1bd0f0c
commit 14a84d3015
5 changed files with 122 additions and 16 deletions

View File

@ -53,3 +53,5 @@
列表仍返回服务端可领取判定;空 sourceValidationStatus 仅表示没有准入阻断,不是完整源证书。任务详情对过渡记录返回 unverified、required=false,不写造假的 fingerprint,不将后台作业改成成功。异常任务保留工作区、草稿、归属及审核历史,不清空、不自动回池;管理员保留异常查询入口。
后台补验/源修复按独立固定清单及生产数据授权执行;补验状态不通过改写旧时间轴成功证据来隐藏失败。实际补验期间进入严格校验的任务可能暂时阻断,须有明确窗口或使用独立受审查的后台审计方案,不能声称无中断。收紧回严格默认前,先核对旧任务和公共池的真实证书准备度;移除配置会重新阻断尚未完成补验的旧任务,不能盲目回退。
旧时间轴或源作业的空指纹不等于已知非空指纹冲突:两条旧 completed 记录都在截止时间前、均非完整重校验、且没有源索引时,可使用同一过渡规则,仍呈现未完整校验。反之,已有匹配当前 outbox、同组织/项目/Run 的完整源作业证明时,不要求旧时间轴重复持有同一指纹;这是真实完整证明,不是源索引流程的豁免。任何失败、活动作业、非空指纹冲突、显式重校验尚未完成仍阻断。

View File

@ -2,6 +2,7 @@ package api
import (
"droboticflow/internal/domain"
"droboticflow/internal/store"
"testing"
)
@ -28,3 +29,33 @@ func TestLegacySourceTransitionStatusIsNotFullProof(t *testing.T) {
t.Fatal("read triggered task writes")
}
}
func TestLegacySourceMetadataGapKeepsUnknownSeparateFromMismatch(t *testing.T) {
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "2026-11-12T00:00:00Z")
f := newCombinedDraftFixture(t)
item := f.item
item.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: "current"}
base := domain.AnnotationIndexJob{ID: f.task.ID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, RunID: f.task.RunID, Status: "completed", CreatedAt: "2026-10-01T00:00:00Z"}
src := base
src.ID = store.AnnotationSourceJobID(f.task.OrganizationID, "current")
wrapped := &sourceProofReadStore{MemoryStore: f.store, timeline: &base, source: &src}
engine := &server{store: wrapped}
v, e := engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true)
if e != nil || v.Status != "unverified" || v.Required || v.blocked() {
t.Fatalf("old metadata-only gap blocked or reported valid: %+v %v", v, e)
}
src.SourceFingerprint = "different"
v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true)
if e != nil || !v.blocked() {
t.Fatalf("real mismatch admitted: %+v %v", v, e)
}
src.SourceFingerprint = "current"
item.ProcessingJob.SourceIndex = true
v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true)
if e != nil || v.Status != "valid" || v.blocked() || !v.Required {
t.Fatalf("matching full source proof ignored: %+v %v", v, e)
}
if wrapped.taskWrites != 0 {
t.Fatal("status read wrote metadata")
}
}

View File

@ -14,9 +14,10 @@ import (
)
type legacyTransitionCase struct {
name string
change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob)
allowed bool
name string
change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob)
allowed bool
fullProof bool
}
func legacyTransitionCases() []legacyTransitionCase {
@ -52,6 +53,23 @@ func legacyTransitionCases() []legacyTransitionCase {
{name: "foreign-timeline", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) {
j.OrganizationID = "foreign"
}},
{name: "old-source-missing-metadata", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) {
v := *j
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
*src = &v
}},
{name: "new-source-missing-metadata", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) {
v := *j
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
v.CreatedAt = "2026-11-12T00:00:00Z"
*src = &v
}},
{name: "revalidated-source-missing-metadata", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) {
v := *j
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
v.RevalidateSource = true
*src = &v
}},
{name: "known-failed-source", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) {
v := *j
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
@ -70,6 +88,13 @@ func legacyTransitionCases() []legacyTransitionCase {
v.SourceFingerprint = "different"
*s = &v
}},
{name: "indexed-existing-complete-proof", fullProof: true, change: func(i *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) {
i.ProcessingJob.SourceIndex = true
v := *j
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
v.SourceFingerprint = "current"
*s = &v
}},
{name: "current-source-proof", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) {
v := *j
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
@ -104,10 +129,10 @@ func TestLegacySourceTransitionFence(t *testing.T) {
if got := LegacyAnnotationSourceCompatible(task, item, &j, src); got != c.allowed {
t.Fatalf("compatibility=%v want=%v", got, c.allowed)
}
if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != c.allowed {
if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != (c.allowed || c.fullProof) {
t.Fatalf("ready=%v want=%v", got, c.allowed)
}
if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != c.allowed {
if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != (c.allowed || c.fullProof) {
t.Fatal("projection differs from eligibility")
}
after, _ := json.Marshal([]any{item, j, src})
@ -162,7 +187,7 @@ func TestLegacySourceTransitionMongoEquivalence(t *testing.T) {
t.Fatal(e)
}
items = append(items, item)
if c.allowed {
if c.allowed || c.fullProof {
expected[item.ID] = true
}
}
@ -190,3 +215,26 @@ func TestLegacySourceTransitionMongoEquivalence(t *testing.T) {
}
}
}
func TestLegacyTimelineAcceptsOnlyMatchingCompleteCurrentSource(t *testing.T) {
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "")
task, item, j, src := legacyTransitionRecords(0, legacyTransitionCases()[0])
item.ProcessingJob.SourceIndex = true
full := j
full.ID = AnnotationSourceJobID(task.OrganizationID, item.ProcessingJob.SourceFingerprint)
full.SourceFingerprint = item.ProcessingJob.SourceFingerprint
src = &full
if !AnnotationItemSourceJobsReady(task, item, &j, src) {
t.Fatal("complete current source proof ignored because legacy timeline lacks metadata")
}
for _, change := range []func(){func() { src.SourceFingerprint = "different" }, func() { src.Status = "failed" }, func() { src.Status = "running" }, func() { j.RevalidateSource = true }, func() { src.RunID = "foreign" }} {
j.RevalidateSource = false
src.SourceFingerprint = item.ProcessingJob.SourceFingerprint
src.Status = "completed"
src.RunID = task.RunID
change()
if AnnotationItemSourceJobsReady(task, item, &j, src) {
t.Fatal("invalid or mismatched proof admitted")
}
}
}

View File

@ -22,11 +22,18 @@ func AnnotationSourceValidationRequired(_ domain.AnnotationTaskRecord) bool {
}
func AnnotationSourceJobsReady(task domain.AnnotationTaskRecord, timeline, source *domain.AnnotationIndexJob) bool {
if timeline == nil || timeline.SourceFingerprint == "" {
if timeline == nil {
return false
}
for _, job := range []*domain.AnnotationIndexJob{timeline, source} {
if job != nil && (job.Status != "completed" || job.SourceFingerprint == "" || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) {
// A complete, current source job is full proof even when the historical
// time-axis job predates the fingerprint field. Explicit revalidation is
// never satisfied by a half-published timeline.
historicalWithSourceProof := !timeline.RevalidateSource && source != nil && source.SourceFingerprint != ""
if timeline.SourceFingerprint == "" && !historicalWithSourceProof {
return false
}
for index, job := range []*domain.AnnotationIndexJob{timeline, source} {
if job != nil && (job.Status != "completed" || (job.SourceFingerprint == "" && !(index == 0 && historicalWithSourceProof)) || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) {
return false
}
}
@ -101,7 +108,16 @@ func LegacyAnnotationSourceCompatible(task domain.AnnotationTaskRecord, item dom
}
}
// A known current-source proof must match the physical outbox identity.
return source == nil || source.SourceFingerprint != "" && item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint
if source == nil {
return true
}
if source.SourceFingerprint != "" {
return item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint
}
// Old completed source-job metadata can lack a fingerprint too. This is
// unknown evidence, not a known mismatch; bound both old records to cutoff.
sourceCreated, err := time.Parse(time.RFC3339Nano, source.CreatedAt)
return !source.RevalidateSource && err == nil && legacySourceCreatedAt.MatchString(source.CreatedAt) && sourceCreated.Before(cutoff)
}
func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool {
@ -212,7 +228,8 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
needsCertificate := bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$processingJob.sourceIndex", true}}, bson.M{"$eq": bson.A{"$subject.payload.materialization.sourceIndex", true}}}}
jobs := bson.M{"$concatArrays": bson.A{"$_sourceTimeline", "$_sourcePrepared"}}
matchingScope := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.organizationId", "$organizationId"}}, bson.M{"$eq": bson.A{"$$job.projectId", "$projectId"}}, bson.M{"$eq": bson.A{"$$job.runId", "$runId"}}}}
safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, nonempty("$$job.sourceFingerprint"), matchingScope}}
historicalSourceProof := bson.M{"$and": bson.A{bson.M{"$not": bson.A{revalidated}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}}
safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, bson.M{"$or": bson.A{nonempty("$$job.sourceFingerprint"), bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.id", "$legacyTaskId"}}, historicalSourceProof}}}}, matchingScope}}
timelinePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTimeline"}, 0}}
taskPresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTask"}, 0}}
certificatePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourcePrepared"}, 0}}
@ -234,7 +251,7 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
ready := bson.M{"$and": bson.A{
taskPresent,
timelinePresent,
nonempty(first("$_sourceTimeline.sourceFingerprint")),
bson.M{"$or": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint")), historicalSourceProof}},
certificateSatisfied,
sourceVersionMatches,
outboxProofMatches,
@ -245,20 +262,24 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
// pending jobs remain excluded; source-index workflows still need proof.
if cutoff, enabled := LegacySourceAdmissionCutoff(); enabled {
created := bson.M{"$convert": bson.M{"input": first("$_sourceTimeline.createdAt"), "to": "date", "onError": nil, "onNull": nil}}
sourceCreated := bson.M{"$convert": bson.M{"input": first("$_sourcePrepared.createdAt"), "to": "date", "onError": nil, "onNull": nil}}
sourceOldDate := bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourcePrepared.createdAt")}, "string"}}, first("$_sourcePrepared.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}}
weakOldSource := bson.M{"$and": bson.A{bson.M{"$not": bson.A{nonempty(first("$_sourcePrepared.sourceFingerprint"))}}, bson.M{"$ne": bson.A{first("$_sourcePrepared.revalidateSource"), true}}, sourceOldDate, bson.M{"$ne": bson.A{sourceCreated, nil}}, bson.M{"$lt": bson.A{sourceCreated, cutoff}}}}
legacySourceMatches := bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, bson.M{"$and": bson.A{nonempty(first("$_sourcePrepared.sourceFingerprint")), sourceVersionMatches}}, weakOldSource}}
legacy := bson.M{"$and": bson.A{taskPresent, timelinePresent,
bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourceTimeline.createdAt")}, "string"}}, first("$_sourceTimeline.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}},
bson.M{"$not": bson.A{revalidated}}, bson.M{"$not": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint"))}},
bson.M{"$ne": bson.A{created, nil}}, bson.M{"$lt": bson.A{created, cutoff}},
bson.M{"$not": bson.A{needsCertificate}}, sourceVersionMatches,
bson.M{"$not": bson.A{needsCertificate}}, legacySourceMatches,
bson.M{"$allElementsTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, matchingScope}}}}}},
bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}},
}}
ready = bson.M{"$or": bson.A{ready, legacy}}
missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{legacy}}}}
}
missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{ready}}}}
active := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$in": bson.A{"$$job.status", bson.A{"queued", "running"}}}}}}}
failed := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$$job.status", "failed"}}, bson.M{"$not": bson.A{matchingScope}}}}}}}}
jobProjection := bson.M{"status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1}
jobProjection := bson.M{"id": 1, "status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1}
lookup := func(collection, local, as string, projection bson.M) bson.D {
return bson.D{{Key: "$lookup", Value: bson.M{"from": collection, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$project": projection}}, "as": as}}}
}

View File

@ -194,7 +194,11 @@
"^TestRandomClaim",
"^TestLegacySourceTransition"
],
"./internal/store": ["^TestSourceAdmissionMemoryClaimFence$", "^TestLegacySourceTransitionFence$"]
"./internal/store": [
"^TestSourceAdmissionMemoryClaimFence$",
"^TestLegacySourceTransitionFence$",
"^TestLegacyTimelineAcceptsOnlyMatchingCompleteCurrentSource$"
]
},
"task": [
{