forked from longtao.wu/droboticflow
🛡️ Distinguish historical proof metadata gaps from source conflicts
This commit is contained in:
parent
a2f1bd0f0c
commit
14a84d3015
@ -53,3 +53,5 @@
|
||||
列表仍返回服务端可领取判定;空 sourceValidationStatus 仅表示没有准入阻断,不是完整源证书。任务详情对过渡记录返回 unverified、required=false,不写造假的 fingerprint,不将后台作业改成成功。异常任务保留工作区、草稿、归属及审核历史,不清空、不自动回池;管理员保留异常查询入口。
|
||||
|
||||
后台补验/源修复按独立固定清单及生产数据授权执行;补验状态不通过改写旧时间轴成功证据来隐藏失败。实际补验期间进入严格校验的任务可能暂时阻断,须有明确窗口或使用独立受审查的后台审计方案,不能声称无中断。收紧回严格默认前,先核对旧任务和公共池的真实证书准备度;移除配置会重新阻断尚未完成补验的旧任务,不能盲目回退。
|
||||
|
||||
旧时间轴或源作业的空指纹不等于已知非空指纹冲突:两条旧 completed 记录都在截止时间前、均非完整重校验、且没有源索引时,可使用同一过渡规则,仍呈现未完整校验。反之,已有匹配当前 outbox、同组织/项目/Run 的完整源作业证明时,不要求旧时间轴重复持有同一指纹;这是真实完整证明,不是源索引流程的豁免。任何失败、活动作业、非空指纹冲突、显式重校验尚未完成仍阻断。
|
||||
|
||||
@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@ -28,3 +29,33 @@ func TestLegacySourceTransitionStatusIsNotFullProof(t *testing.T) {
|
||||
t.Fatal("read triggered task writes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacySourceMetadataGapKeepsUnknownSeparateFromMismatch(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "2026-11-12T00:00:00Z")
|
||||
f := newCombinedDraftFixture(t)
|
||||
item := f.item
|
||||
item.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: "current"}
|
||||
base := domain.AnnotationIndexJob{ID: f.task.ID, OrganizationID: f.task.OrganizationID, ProjectID: f.task.ProjectID, RunID: f.task.RunID, Status: "completed", CreatedAt: "2026-10-01T00:00:00Z"}
|
||||
src := base
|
||||
src.ID = store.AnnotationSourceJobID(f.task.OrganizationID, "current")
|
||||
wrapped := &sourceProofReadStore{MemoryStore: f.store, timeline: &base, source: &src}
|
||||
engine := &server{store: wrapped}
|
||||
v, e := engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true)
|
||||
if e != nil || v.Status != "unverified" || v.Required || v.blocked() {
|
||||
t.Fatalf("old metadata-only gap blocked or reported valid: %+v %v", v, e)
|
||||
}
|
||||
src.SourceFingerprint = "different"
|
||||
v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true)
|
||||
if e != nil || !v.blocked() {
|
||||
t.Fatalf("real mismatch admitted: %+v %v", v, e)
|
||||
}
|
||||
src.SourceFingerprint = "current"
|
||||
item.ProcessingJob.SourceIndex = true
|
||||
v, e = engine.annotationSourceValidationForTask(f.ctx, f.task, &item, true)
|
||||
if e != nil || v.Status != "valid" || v.blocked() || !v.Required {
|
||||
t.Fatalf("matching full source proof ignored: %+v %v", v, e)
|
||||
}
|
||||
if wrapped.taskWrites != 0 {
|
||||
t.Fatal("status read wrote metadata")
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,9 +14,10 @@ import (
|
||||
)
|
||||
|
||||
type legacyTransitionCase struct {
|
||||
name string
|
||||
change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob)
|
||||
allowed bool
|
||||
name string
|
||||
change func(*domain.HumanItemRecord, *domain.AnnotationIndexJob, **domain.AnnotationIndexJob)
|
||||
allowed bool
|
||||
fullProof bool
|
||||
}
|
||||
|
||||
func legacyTransitionCases() []legacyTransitionCase {
|
||||
@ -52,6 +53,23 @@ func legacyTransitionCases() []legacyTransitionCase {
|
||||
{name: "foreign-timeline", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, _ **domain.AnnotationIndexJob) {
|
||||
j.OrganizationID = "foreign"
|
||||
}},
|
||||
{name: "old-source-missing-metadata", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) {
|
||||
v := *j
|
||||
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
|
||||
*src = &v
|
||||
}},
|
||||
{name: "new-source-missing-metadata", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) {
|
||||
v := *j
|
||||
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
|
||||
v.CreatedAt = "2026-11-12T00:00:00Z"
|
||||
*src = &v
|
||||
}},
|
||||
{name: "revalidated-source-missing-metadata", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, src **domain.AnnotationIndexJob) {
|
||||
v := *j
|
||||
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
|
||||
v.RevalidateSource = true
|
||||
*src = &v
|
||||
}},
|
||||
{name: "known-failed-source", change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) {
|
||||
v := *j
|
||||
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
|
||||
@ -70,6 +88,13 @@ func legacyTransitionCases() []legacyTransitionCase {
|
||||
v.SourceFingerprint = "different"
|
||||
*s = &v
|
||||
}},
|
||||
{name: "indexed-existing-complete-proof", fullProof: true, change: func(i *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) {
|
||||
i.ProcessingJob.SourceIndex = true
|
||||
v := *j
|
||||
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
|
||||
v.SourceFingerprint = "current"
|
||||
*s = &v
|
||||
}},
|
||||
{name: "current-source-proof", allowed: true, change: func(_ *domain.HumanItemRecord, j *domain.AnnotationIndexJob, s **domain.AnnotationIndexJob) {
|
||||
v := *j
|
||||
v.ID = AnnotationSourceJobID(j.OrganizationID, "current")
|
||||
@ -104,10 +129,10 @@ func TestLegacySourceTransitionFence(t *testing.T) {
|
||||
if got := LegacyAnnotationSourceCompatible(task, item, &j, src); got != c.allowed {
|
||||
t.Fatalf("compatibility=%v want=%v", got, c.allowed)
|
||||
}
|
||||
if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != c.allowed {
|
||||
if got := AnnotationItemSourceJobsReady(task, item, &j, src); got != (c.allowed || c.fullProof) {
|
||||
t.Fatalf("ready=%v want=%v", got, c.allowed)
|
||||
}
|
||||
if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != c.allowed {
|
||||
if (AnnotationSourceAdmissionStatus(task, item, &j, src) == "") != (c.allowed || c.fullProof) {
|
||||
t.Fatal("projection differs from eligibility")
|
||||
}
|
||||
after, _ := json.Marshal([]any{item, j, src})
|
||||
@ -162,7 +187,7 @@ func TestLegacySourceTransitionMongoEquivalence(t *testing.T) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items = append(items, item)
|
||||
if c.allowed {
|
||||
if c.allowed || c.fullProof {
|
||||
expected[item.ID] = true
|
||||
}
|
||||
}
|
||||
@ -190,3 +215,26 @@ func TestLegacySourceTransitionMongoEquivalence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyTimelineAcceptsOnlyMatchingCompleteCurrentSource(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "")
|
||||
task, item, j, src := legacyTransitionRecords(0, legacyTransitionCases()[0])
|
||||
item.ProcessingJob.SourceIndex = true
|
||||
full := j
|
||||
full.ID = AnnotationSourceJobID(task.OrganizationID, item.ProcessingJob.SourceFingerprint)
|
||||
full.SourceFingerprint = item.ProcessingJob.SourceFingerprint
|
||||
src = &full
|
||||
if !AnnotationItemSourceJobsReady(task, item, &j, src) {
|
||||
t.Fatal("complete current source proof ignored because legacy timeline lacks metadata")
|
||||
}
|
||||
for _, change := range []func(){func() { src.SourceFingerprint = "different" }, func() { src.Status = "failed" }, func() { src.Status = "running" }, func() { j.RevalidateSource = true }, func() { src.RunID = "foreign" }} {
|
||||
j.RevalidateSource = false
|
||||
src.SourceFingerprint = item.ProcessingJob.SourceFingerprint
|
||||
src.Status = "completed"
|
||||
src.RunID = task.RunID
|
||||
change()
|
||||
if AnnotationItemSourceJobsReady(task, item, &j, src) {
|
||||
t.Fatal("invalid or mismatched proof admitted")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,11 +22,18 @@ func AnnotationSourceValidationRequired(_ domain.AnnotationTaskRecord) bool {
|
||||
}
|
||||
|
||||
func AnnotationSourceJobsReady(task domain.AnnotationTaskRecord, timeline, source *domain.AnnotationIndexJob) bool {
|
||||
if timeline == nil || timeline.SourceFingerprint == "" {
|
||||
if timeline == nil {
|
||||
return false
|
||||
}
|
||||
for _, job := range []*domain.AnnotationIndexJob{timeline, source} {
|
||||
if job != nil && (job.Status != "completed" || job.SourceFingerprint == "" || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) {
|
||||
// A complete, current source job is full proof even when the historical
|
||||
// time-axis job predates the fingerprint field. Explicit revalidation is
|
||||
// never satisfied by a half-published timeline.
|
||||
historicalWithSourceProof := !timeline.RevalidateSource && source != nil && source.SourceFingerprint != ""
|
||||
if timeline.SourceFingerprint == "" && !historicalWithSourceProof {
|
||||
return false
|
||||
}
|
||||
for index, job := range []*domain.AnnotationIndexJob{timeline, source} {
|
||||
if job != nil && (job.Status != "completed" || (job.SourceFingerprint == "" && !(index == 0 && historicalWithSourceProof)) || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@ -101,7 +108,16 @@ func LegacyAnnotationSourceCompatible(task domain.AnnotationTaskRecord, item dom
|
||||
}
|
||||
}
|
||||
// A known current-source proof must match the physical outbox identity.
|
||||
return source == nil || source.SourceFingerprint != "" && item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint
|
||||
if source == nil {
|
||||
return true
|
||||
}
|
||||
if source.SourceFingerprint != "" {
|
||||
return item.ProcessingJob != nil && source.SourceFingerprint == item.ProcessingJob.SourceFingerprint
|
||||
}
|
||||
// Old completed source-job metadata can lack a fingerprint too. This is
|
||||
// unknown evidence, not a known mismatch; bound both old records to cutoff.
|
||||
sourceCreated, err := time.Parse(time.RFC3339Nano, source.CreatedAt)
|
||||
return !source.RevalidateSource && err == nil && legacySourceCreatedAt.MatchString(source.CreatedAt) && sourceCreated.Before(cutoff)
|
||||
}
|
||||
|
||||
func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool {
|
||||
@ -212,7 +228,8 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
|
||||
needsCertificate := bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$processingJob.sourceIndex", true}}, bson.M{"$eq": bson.A{"$subject.payload.materialization.sourceIndex", true}}}}
|
||||
jobs := bson.M{"$concatArrays": bson.A{"$_sourceTimeline", "$_sourcePrepared"}}
|
||||
matchingScope := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.organizationId", "$organizationId"}}, bson.M{"$eq": bson.A{"$$job.projectId", "$projectId"}}, bson.M{"$eq": bson.A{"$$job.runId", "$runId"}}}}
|
||||
safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, nonempty("$$job.sourceFingerprint"), matchingScope}}
|
||||
historicalSourceProof := bson.M{"$and": bson.A{bson.M{"$not": bson.A{revalidated}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}}
|
||||
safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, bson.M{"$or": bson.A{nonempty("$$job.sourceFingerprint"), bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.id", "$legacyTaskId"}}, historicalSourceProof}}}}, matchingScope}}
|
||||
timelinePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTimeline"}, 0}}
|
||||
taskPresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTask"}, 0}}
|
||||
certificatePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourcePrepared"}, 0}}
|
||||
@ -234,7 +251,7 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
|
||||
ready := bson.M{"$and": bson.A{
|
||||
taskPresent,
|
||||
timelinePresent,
|
||||
nonempty(first("$_sourceTimeline.sourceFingerprint")),
|
||||
bson.M{"$or": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint")), historicalSourceProof}},
|
||||
certificateSatisfied,
|
||||
sourceVersionMatches,
|
||||
outboxProofMatches,
|
||||
@ -245,20 +262,24 @@ func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
|
||||
// pending jobs remain excluded; source-index workflows still need proof.
|
||||
if cutoff, enabled := LegacySourceAdmissionCutoff(); enabled {
|
||||
created := bson.M{"$convert": bson.M{"input": first("$_sourceTimeline.createdAt"), "to": "date", "onError": nil, "onNull": nil}}
|
||||
sourceCreated := bson.M{"$convert": bson.M{"input": first("$_sourcePrepared.createdAt"), "to": "date", "onError": nil, "onNull": nil}}
|
||||
sourceOldDate := bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourcePrepared.createdAt")}, "string"}}, first("$_sourcePrepared.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}}
|
||||
weakOldSource := bson.M{"$and": bson.A{bson.M{"$not": bson.A{nonempty(first("$_sourcePrepared.sourceFingerprint"))}}, bson.M{"$ne": bson.A{first("$_sourcePrepared.revalidateSource"), true}}, sourceOldDate, bson.M{"$ne": bson.A{sourceCreated, nil}}, bson.M{"$lt": bson.A{sourceCreated, cutoff}}}}
|
||||
legacySourceMatches := bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, bson.M{"$and": bson.A{nonempty(first("$_sourcePrepared.sourceFingerprint")), sourceVersionMatches}}, weakOldSource}}
|
||||
legacy := bson.M{"$and": bson.A{taskPresent, timelinePresent,
|
||||
bson.M{"$regexMatch": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": first("$_sourceTimeline.createdAt")}, "string"}}, first("$_sourceTimeline.createdAt"), ""}}, "regex": legacySourceCreatedAt.String()}},
|
||||
bson.M{"$not": bson.A{revalidated}}, bson.M{"$not": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint"))}},
|
||||
bson.M{"$ne": bson.A{created, nil}}, bson.M{"$lt": bson.A{created, cutoff}},
|
||||
bson.M{"$not": bson.A{needsCertificate}}, sourceVersionMatches,
|
||||
bson.M{"$not": bson.A{needsCertificate}}, legacySourceMatches,
|
||||
bson.M{"$allElementsTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, matchingScope}}}}}},
|
||||
bson.M{"$or": bson.A{bson.M{"$not": bson.A{certificatePresent}}, nonempty(first("$_sourcePrepared.sourceFingerprint"))}},
|
||||
}}
|
||||
ready = bson.M{"$or": bson.A{ready, legacy}}
|
||||
missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{legacy}}}}
|
||||
}
|
||||
missing = bson.M{"$and": bson.A{missing, bson.M{"$not": bson.A{ready}}}}
|
||||
active := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$in": bson.A{"$$job.status", bson.A{"queued", "running"}}}}}}}
|
||||
failed := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$$job.status", "failed"}}, bson.M{"$not": bson.A{matchingScope}}}}}}}}
|
||||
jobProjection := bson.M{"status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1}
|
||||
jobProjection := bson.M{"id": 1, "status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1, "createdAt": 1}
|
||||
lookup := func(collection, local, as string, projection bson.M) bson.D {
|
||||
return bson.D{{Key: "$lookup", Value: bson.M{"from": collection, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$project": projection}}, "as": as}}}
|
||||
}
|
||||
|
||||
@ -194,7 +194,11 @@
|
||||
"^TestRandomClaim",
|
||||
"^TestLegacySourceTransition"
|
||||
],
|
||||
"./internal/store": ["^TestSourceAdmissionMemoryClaimFence$", "^TestLegacySourceTransitionFence$"]
|
||||
"./internal/store": [
|
||||
"^TestSourceAdmissionMemoryClaimFence$",
|
||||
"^TestLegacySourceTransitionFence$",
|
||||
"^TestLegacyTimelineAcceptsOnlyMatchingCompleteCurrentSource$"
|
||||
]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user