🛡️ Bound source validation to explicit current requests

This commit is contained in:
d-robotics 2026-10-17 10:38:53 +08:00
parent 3a4e4d308a
commit ba6c46cffa
5 changed files with 99 additions and 3 deletions

View File

@ -14,6 +14,8 @@
2. 每个 Run 的源版本第一次使用时,完整验证双目解码帧数、媒体时钟、PTS 顺序、共享边界、IMU 有效配对。原始硬件时间允许有间隔,不把硬件时间当作视频播放时间。
3. 校验后将两路视频、两份 PTS、一份 IMU 固定到 `system/runs/<run>/annotation-sources/<fingerprint>/`。BOS 使用条件复制,不能覆盖已有固定对象;本地使用完整临时文件及原子发布。源版本变化必须重新验证,旧修订继续引用自己的固定版本。
4. 证书以校验和、gzip 保存于既有 timeline 索引集合。独立持久化校验队列负责冷源;轻量修订等待时让出执行槽,不消耗崩溃重试次数。校验租约续期,超时可恢复。
旧时间轴队列只建立原有索引,不因启用新校验器而生成固定源副本。当前用户确认创建的 `source-v1:` 请求优先于历史时间轴工作,使用现有 ID/状态索引,不新增迁移或改写旧队列记录。
5. 后续每个修订只写修订清单、每片 annotation 和 synchronization JSON。记录半开硬件时间窗、双目各自帧区间及播放偏移,保留完整覆盖、舍弃理由、动作与审核冻结校验。
6. 最终验收使用已固定的完整源媒体和切片逻辑时间映射;不依赖重新生成的逐片视频、PTS、IMU。切片本地时间和双目源时间分别转换,元数据就绪后才挂载播放器。
7. 旧目录交付仍在终审之后按帧生成保留切片媒体和 CSV,舍弃区间不产生交付媒体。该阶段仍有转码成本,与交互式切片队列分离。

View File

@ -185,7 +185,9 @@ func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppC
}
if readErr == nil && task != nil {
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
if readErr == nil && s.annotationSourceIndex {
// Legacy timeline jobs may predate this release. They must
// not silently trigger full-media validation/source copies.
if readErr == nil && s.annotationSourceIndex && j.TaskID != "" {
run, err := db.GetRun(indexCtx, task.RunID)
if err != nil {
readErr = err

View File

@ -5,6 +5,8 @@ import (
"droboticflow/internal/config"
"droboticflow/internal/domain"
"net/http"
"os"
"path/filepath"
"testing"
"time"
)
@ -68,3 +70,30 @@ func TestSourceIndexColdValidationDoesNotFailOrOccupyRevisionWorker(t *testing.T
}
t.Fatal("deferred task did not resume after validation/restart")
}
func TestSourceIndexLegacyTimelineQueueDoesNotBackfillSourceCopies(t *testing.T) {
t.Setenv("ANNOTATION_SOURCE_INDEX", "true")
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation")
f, root := annotationProcessorFixture(t)
ctx, cancel := context.WithCancel(f.ctx)
done := make(chan error, 1)
go func() {
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
}()
defer func() { cancel(); <-done }()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
job, err := f.store.GetAnnotationIndexJob(ctx, f.task.ID)
if err != nil {
t.Fatal(err)
}
if job != nil && job.Status == "completed" {
if _, err := os.Stat(filepath.Join(root, "system", "runs", f.task.RunID, "annotation-sources")); !os.IsNotExist(err) {
t.Fatal("legacy timeline job triggered source-copy backfill", err)
}
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("legacy timeline job did not complete")
}

View File

@ -7,6 +7,7 @@ import (
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"sort"
"strings"
)
type AnnotationProcessingStore interface {
@ -107,7 +108,13 @@ func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until
ids = append(ids, id)
}
}
sort.Strings(ids)
sort.Slice(ids, func(i, j int) bool {
a, b := strings.HasPrefix(ids[i], "source-v1:"), strings.HasPrefix(ids[j], "source-v1:")
if a != b {
return a
}
return ids[i] < ids[j]
})
if len(ids) == 0 {
return nil, nil
}
@ -121,7 +128,19 @@ func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until
}
func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
var j domain.AnnotationIndexJob
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&j)
filter := bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}
update := bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}
opts := options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)
// Current explicit source validation outranks historical timeline work.
// The bounded prefix range also works with the existing unique id index;
// no schema migration or historical record rewrite is required.
filter["id"] = bson.M{"$gte": "source-v1:", "$lt": "source-v1;"}
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
if e != mongo.ErrNoDocuments {
return &j, e
}
delete(filter, "id")
e = s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
if e == mongo.ErrNoDocuments {
return nil, nil
}

View File

@ -68,3 +68,47 @@ func TestSourceIndexJobRecoveryFencesStaleOwnerAndPreservesLegacyJobs(t *testing
t.Fatal("source retry changed legacy job")
}
}
func TestSourceIndexJobPriorityBeforeLegacyTimeline(t *testing.T) {
s := NewMemoryStore()
ctx := t.Context()
old := domain.AnnotationTaskRecord{ID: "a-old-timeline", OrganizationID: "org", RunID: "run"}
current := old
current.ID = "current-confirmation"
if err := s.EnqueueAnnotationIndex(ctx, old); err != nil {
t.Fatal(err)
}
if err := s.EnqueueAnnotationSourceIndex(ctx, current, "fingerprint", false); err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
j, err := s.ClaimAnnotationIndex(ctx, "owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if err != nil || j == nil || j.TaskID != current.ID {
t.Fatal("current source request queued behind legacy timeline work", j, err)
}
}
func TestSourceIndexJobPriorityMongoBeforeLegacyTimeline(t *testing.T) {
s, ctx := projectStripeTestStore(t)
if err := s.EnsureIndexes(ctx); err != nil {
t.Fatal(err)
}
old := domain.AnnotationTaskRecord{ID: "a-old-timeline", OrganizationID: "org", RunID: "run"}
current := old
current.ID = "current-confirmation"
if err := s.EnqueueAnnotationIndex(ctx, old); err != nil {
t.Fatal(err)
}
if err := s.EnqueueAnnotationSourceIndex(ctx, current, "fingerprint", false); err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
j, err := s.ClaimAnnotationIndex(ctx, "owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if err != nil || j == nil || j.TaskID != current.ID {
t.Fatal("Mongo current request queued behind historical work", j, err)
}
next, err := s.ClaimAnnotationIndex(ctx, "other", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if err != nil || next == nil || next.ID != old.ID {
t.Fatal("source request claimed twice or legacy work starved", next, err)
}
}