forked from longtao.wu/droboticflow
🛡️ Bound source validation to explicit current requests
This commit is contained in:
parent
3a4e4d308a
commit
ba6c46cffa
@ -14,6 +14,8 @@
|
||||
2. 每个 Run 的源版本第一次使用时,完整验证双目解码帧数、媒体时钟、PTS 顺序、共享边界、IMU 有效配对。原始硬件时间允许有间隔,不把硬件时间当作视频播放时间。
|
||||
3. 校验后将两路视频、两份 PTS、一份 IMU 固定到 `system/runs/<run>/annotation-sources/<fingerprint>/`。BOS 使用条件复制,不能覆盖已有固定对象;本地使用完整临时文件及原子发布。源版本变化必须重新验证,旧修订继续引用自己的固定版本。
|
||||
4. 证书以校验和、gzip 保存于既有 timeline 索引集合。独立持久化校验队列负责冷源;轻量修订等待时让出执行槽,不消耗崩溃重试次数。校验租约续期,超时可恢复。
|
||||
|
||||
旧时间轴队列只建立原有索引,不因启用新校验器而生成固定源副本。当前用户确认创建的 `source-v1:` 请求优先于历史时间轴工作,使用现有 ID/状态索引,不新增迁移或改写旧队列记录。
|
||||
5. 后续每个修订只写修订清单、每片 annotation 和 synchronization JSON。记录半开硬件时间窗、双目各自帧区间及播放偏移,保留完整覆盖、舍弃理由、动作与审核冻结校验。
|
||||
6. 最终验收使用已固定的完整源媒体和切片逻辑时间映射;不依赖重新生成的逐片视频、PTS、IMU。切片本地时间和双目源时间分别转换,元数据就绪后才挂载播放器。
|
||||
7. 旧目录交付仍在终审之后按帧生成保留切片媒体和 CSV,舍弃区间不产生交付媒体。该阶段仍有转码成本,与交互式切片队列分离。
|
||||
|
||||
@ -185,7 +185,9 @@ func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppC
|
||||
}
|
||||
if readErr == nil && task != nil {
|
||||
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
|
||||
if readErr == nil && s.annotationSourceIndex {
|
||||
// Legacy timeline jobs may predate this release. They must
|
||||
// not silently trigger full-media validation/source copies.
|
||||
if readErr == nil && s.annotationSourceIndex && j.TaskID != "" {
|
||||
run, err := db.GetRun(indexCtx, task.RunID)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
|
||||
@ -5,6 +5,8 @@ import (
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -68,3 +70,30 @@ func TestSourceIndexColdValidationDoesNotFailOrOccupyRevisionWorker(t *testing.T
|
||||
}
|
||||
t.Fatal("deferred task did not resume after validation/restart")
|
||||
}
|
||||
|
||||
func TestSourceIndexLegacyTimelineQueueDoesNotBackfillSourceCopies(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_SOURCE_INDEX", "true")
|
||||
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
|
||||
}()
|
||||
defer func() { cancel(); <-done }()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
job, err := f.store.GetAnnotationIndexJob(ctx, f.task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job != nil && job.Status == "completed" {
|
||||
if _, err := os.Stat(filepath.Join(root, "system", "runs", f.task.RunID, "annotation-sources")); !os.IsNotExist(err) {
|
||||
t.Fatal("legacy timeline job triggered source-copy backfill", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("legacy timeline job did not complete")
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type AnnotationProcessingStore interface {
|
||||
@ -107,7 +108,13 @@ func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
sort.Slice(ids, func(i, j int) bool {
|
||||
a, b := strings.HasPrefix(ids[i], "source-v1:"), strings.HasPrefix(ids[j], "source-v1:")
|
||||
if a != b {
|
||||
return a
|
||||
}
|
||||
return ids[i] < ids[j]
|
||||
})
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@ -121,7 +128,19 @@ func (s *MemoryStore) ClaimAnnotationIndex(_ context.Context, owner, now, until
|
||||
}
|
||||
func (s *MongoStore) ClaimAnnotationIndex(ctx context.Context, owner, now, until string) (*domain.AnnotationIndexJob, error) {
|
||||
var j domain.AnnotationIndexJob
|
||||
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}, bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}, options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)).Decode(&j)
|
||||
filter := bson.M{"$or": bson.A{bson.M{"status": "queued", "nextAttemptAt": bson.M{"$lte": now}}, bson.M{"status": "running", "leaseUntil": bson.M{"$lt": now}}}}
|
||||
update := bson.M{"$set": bson.M{"status": "running", "owner": owner, "leaseUntil": until}, "$inc": bson.M{"attempt": 1}}
|
||||
opts := options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After)
|
||||
// Current explicit source validation outranks historical timeline work.
|
||||
// The bounded prefix range also works with the existing unique id index;
|
||||
// no schema migration or historical record rewrite is required.
|
||||
filter["id"] = bson.M{"$gte": "source-v1:", "$lt": "source-v1;"}
|
||||
e := s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
|
||||
if e != mongo.ErrNoDocuments {
|
||||
return &j, e
|
||||
}
|
||||
delete(filter, "id")
|
||||
e = s.db.Collection("annotationIndexJobs").FindOneAndUpdate(ctx, filter, update, opts).Decode(&j)
|
||||
if e == mongo.ErrNoDocuments {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@ -68,3 +68,47 @@ func TestSourceIndexJobRecoveryFencesStaleOwnerAndPreservesLegacyJobs(t *testing
|
||||
t.Fatal("source retry changed legacy job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceIndexJobPriorityBeforeLegacyTimeline(t *testing.T) {
|
||||
s := NewMemoryStore()
|
||||
ctx := t.Context()
|
||||
old := domain.AnnotationTaskRecord{ID: "a-old-timeline", OrganizationID: "org", RunID: "run"}
|
||||
current := old
|
||||
current.ID = "current-confirmation"
|
||||
if err := s.EnqueueAnnotationIndex(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueueAnnotationSourceIndex(ctx, current, "fingerprint", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
j, err := s.ClaimAnnotationIndex(ctx, "owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || j == nil || j.TaskID != current.ID {
|
||||
t.Fatal("current source request queued behind legacy timeline work", j, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceIndexJobPriorityMongoBeforeLegacyTimeline(t *testing.T) {
|
||||
s, ctx := projectStripeTestStore(t)
|
||||
if err := s.EnsureIndexes(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := domain.AnnotationTaskRecord{ID: "a-old-timeline", OrganizationID: "org", RunID: "run"}
|
||||
current := old
|
||||
current.ID = "current-confirmation"
|
||||
if err := s.EnqueueAnnotationIndex(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.EnqueueAnnotationSourceIndex(ctx, current, "fingerprint", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
j, err := s.ClaimAnnotationIndex(ctx, "owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || j == nil || j.TaskID != current.ID {
|
||||
t.Fatal("Mongo current request queued behind historical work", j, err)
|
||||
}
|
||||
next, err := s.ClaimAnnotationIndex(ctx, "other", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || next == nil || next.ID != old.ID {
|
||||
t.Fatal("source request claimed twice or legacy work starved", next, err)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user