diff --git a/docs/adr/0010-source-backed-slice-revisions.md b/docs/adr/0010-source-backed-slice-revisions.md index 43f6cd02..3972de9f 100644 --- a/docs/adr/0010-source-backed-slice-revisions.md +++ b/docs/adr/0010-source-backed-slice-revisions.md @@ -14,6 +14,8 @@ 2. 每个 Run 的源版本第一次使用时,完整验证双目解码帧数、媒体时钟、PTS 顺序、共享边界、IMU 有效配对。原始硬件时间允许有间隔,不把硬件时间当作视频播放时间。 3. 校验后将两路视频、两份 PTS、一份 IMU 固定到 `system/runs//annotation-sources//`。BOS 使用条件复制,不能覆盖已有固定对象;本地使用完整临时文件及原子发布。源版本变化必须重新验证,旧修订继续引用自己的固定版本。 4. 证书以校验和、gzip 保存于既有 timeline 索引集合。独立持久化校验队列负责冷源;轻量修订等待时让出执行槽,不消耗崩溃重试次数。校验租约续期,超时可恢复。 + + 旧时间轴队列只建立原有索引,不因启用新校验器而生成固定源副本。当前用户确认创建的 `source-v1:` 请求优先于历史时间轴工作,使用现有 ID/状态索引,不新增迁移或改写旧队列记录。 5. 后续每个修订只写修订清单、每片 annotation 和 synchronization JSON。记录半开硬件时间窗、双目各自帧区间及播放偏移,保留完整覆盖、舍弃理由、动作与审核冻结校验。 6. 最终验收使用已固定的完整源媒体和切片逻辑时间映射;不依赖重新生成的逐片视频、PTS、IMU。切片本地时间和双目源时间分别转换,元数据就绪后才挂载播放器。 7. 旧目录交付仍在终审之后按帧生成保留切片媒体和 CSV,舍弃区间不产生交付媒体。该阶段仍有转码成本,与交互式切片队列分离。 diff --git a/internal/api/annotation_processor.go b/internal/api/annotation_processor.go index 4a35052c..13725ee0 100644 --- a/internal/api/annotation_processor.go +++ b/internal/api/annotation_processor.go @@ -185,7 +185,9 @@ func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppC } if readErr == nil && task != nil { _, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left") - if readErr == nil && s.annotationSourceIndex { + // Legacy timeline jobs may predate this release. They must + // not silently trigger full-media validation/source copies. + if readErr == nil && s.annotationSourceIndex && j.TaskID != "" { run, err := db.GetRun(indexCtx, task.RunID) if err != nil { readErr = err diff --git a/internal/api/annotation_source_queue_test.go b/internal/api/annotation_source_queue_test.go index f65bd225..a25d4934 100644 --- a/internal/api/annotation_source_queue_test.go +++ b/internal/api/annotation_source_queue_test.go @@ -5,6 +5,8 @@ import ( "droboticflow/internal/config" "droboticflow/internal/domain" "net/http" + "os" + "path/filepath" "testing" "time" ) @@ -68,3 +70,30 @@ func TestSourceIndexColdValidationDoesNotFailOrOccupyRevisionWorker(t *testing.T } t.Fatal("deferred task did not resume after validation/restart") } + +func TestSourceIndexLegacyTimelineQueueDoesNotBackfillSourceCopies(t *testing.T) { + t.Setenv("ANNOTATION_SOURCE_INDEX", "true") + t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation") + f, root := annotationProcessorFixture(t) + ctx, cancel := context.WithCancel(f.ctx) + done := make(chan error, 1) + go func() { + done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false) + }() + defer func() { cancel(); <-done }() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + job, err := f.store.GetAnnotationIndexJob(ctx, f.task.ID) + if err != nil { + t.Fatal(err) + } + if job != nil && job.Status == "completed" { + if _, err := os.Stat(filepath.Join(root, "system", "runs", f.task.RunID, "annotation-sources")); !os.IsNotExist(err) { + t.Fatal("legacy timeline job triggered source-copy backfill", err) + } + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("legacy timeline job did not complete") +} diff --git a/internal/store/annotation_processing.go b/internal/store/annotation_processing.go index 56a75649..cae5de6e 100644 --- a/internal/store/annotation_processing.go +++ b/internal/store/annotation_processing.go @@ -7,6 +7,7 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "sort" + "strings" ) type AnnotationProcessingStore interface { @@ -107,7 +108,13 @@ func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until ids = append(ids, id) } } - sort.Strings(ids) + sort.Slice(ids, func(i, j int) bool { + a, b := strings.HasPrefix(ids[i], "source-v1:"), strings.HasPrefix(ids[j], "source-v1:") + if a != b { + return a + } + return ids[i] < ids[j] + }) if len(ids) == 0 { return nil, nil } @@ -121,7 +128,19 @@ func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until } func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) { var j domain.AnnotationIndexJob - e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&j) + filter := bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}} + update := bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}} + opts := options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After) + // Current explicit source validation outranks historical timeline work. + // The bounded prefix range also works with the existing unique id index; + // no schema migration or historical record rewrite is required. + filter["id"] = bson.M{"$gte": "source-v1:", "$lt": "source-v1;"} + e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j) + if e != mongo.ErrNoDocuments { + return &j, e + } + delete(filter, "id") + e = s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j) if e == mongo.ErrNoDocuments { return nil, nil } diff --git a/internal/store/annotation_source_jobs_test.go b/internal/store/annotation_source_jobs_test.go index 2a130e70..01d02bf0 100644 --- a/internal/store/annotation_source_jobs_test.go +++ b/internal/store/annotation_source_jobs_test.go @@ -68,3 +68,47 @@ func TestSourceIndexJobRecoveryFencesStaleOwnerAndPreservesLegacyJobs(t *testing t.Fatal("source retry changed legacy job") } } + +func TestSourceIndexJobPriorityBeforeLegacyTimeline(t *testing.T) { + s := NewMemoryStore() + ctx := t.Context() + old := domain.AnnotationTaskRecord{ID: "a-old-timeline", OrganizationID: "org", RunID: "run"} + current := old + current.ID = "current-confirmation" + if err := s.EnqueueAnnotationIndex(ctx, old); err != nil { + t.Fatal(err) + } + if err := s.EnqueueAnnotationSourceIndex(ctx, current, "fingerprint", false); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + j, err := s.ClaimAnnotationIndex(ctx, "owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) + if err != nil || j == nil || j.TaskID != current.ID { + t.Fatal("current source request queued behind legacy timeline work", j, err) + } +} + +func TestSourceIndexJobPriorityMongoBeforeLegacyTimeline(t *testing.T) { + s, ctx := projectStripeTestStore(t) + if err := s.EnsureIndexes(ctx); err != nil { + t.Fatal(err) + } + old := domain.AnnotationTaskRecord{ID: "a-old-timeline", OrganizationID: "org", RunID: "run"} + current := old + current.ID = "current-confirmation" + if err := s.EnqueueAnnotationIndex(ctx, old); err != nil { + t.Fatal(err) + } + if err := s.EnqueueAnnotationSourceIndex(ctx, current, "fingerprint", false); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + j, err := s.ClaimAnnotationIndex(ctx, "owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) + if err != nil || j == nil || j.TaskID != current.ID { + t.Fatal("Mongo current request queued behind historical work", j, err) + } + next, err := s.ClaimAnnotationIndex(ctx, "other", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano)) + if err != nil || next == nil || next.ID != old.ID { + t.Fatal("source request claimed twice or legacy work starved", next, err) + } +}