From 116a162aaf396ad79e5e3d8cf00005072dda00b3 Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 23 Sep 2026 02:25:56 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9A=A1=20Remove=20per-target=20re-valida?= =?UTF-8?q?tion=20redundancy=20in=20BOS=20deletion=20execution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deleteProjectTarget's "bos" case called validateDeletionReferences again for every single BOS target — with SL-data's ~2,600 BOS targets, that meant re-running the (already expensive, now individually fast but still ~10s including Mongo I/O) org-wide reference check up to 2,600 times during actual deletion execution, not just once during preview. Caught live: SL-data's real deletion run was in the BOS cleanup phase at ~1.5 items/minute, which would have taken roughly 29 hours to finish. executeProjectDeletion already re-runs validateDeletionReferences once, right before it starts processing non-runtime targets (the runtimeDone gate at line ~724) — that is the correct point to catch a run created between plan-build and execution referencing a path about to be deleted. Repeating it per-target adds no additional safety, since nothing about processing one BOS target changes what other runs reference within the same execution pass. The only other caller (supplier_deletion.go:442) passes an empty ProjectDeletionPlan{}, so the removed call was already a no-op there. Co-Authored-By: Claude Sonnet 5 --- internal/api/project_deletion.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/api/project_deletion.go b/internal/api/project_deletion.go index 094e1d68..e31033cc 100644 --- a/internal/api/project_deletion.go +++ b/internal/api/project_deletion.go @@ -452,9 +452,14 @@ func (s *server) deleteProjectTarget(ctx context.Context, plan domain.ProjectDel } return nil case "bos": - if err := s.validateDeletionReferences(ctx, plan); err != nil { - return err - } + // validateDeletionReferences is already re-run once by + // executeProjectDeletion immediately before it starts processing + // non-runtime targets (the runtimeDone gate) — that is the right + // point to catch a run created between plan-build and execution + // referencing this path. Re-running the whole org-wide validation + // again here, per bos target, made a project with a few thousand + // BOS targets take hours (each call re-fetches and re-scans every + // other run in the organization). browser, conn, cleanup, err := s.resolveObjectStorageBrowser(ctx, t.ConnectionID) if err != nil { return err From 2864d2aa1143bd794bf23f2e2d08df8593a18249 Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 23 Sep 2026 04:30:13 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E2=9A=A1=20Process=20deletion=20targets=20?= =?UTF-8?q?concurrently=20within=20each=20phase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit executeProjectDeletion processed every target (runtime, then workspace, then bos, then records) strictly one at a time — for a project with thousands of real runs, that meant thousands of sequential K8s/BOS/Mongo round trips. Observed live on Fanyu: BLF-data's runtime phase (stopping 4,641 runs) took tens of minutes at that pace. Replaced the per-target sequential loop with a bounded worker pool (PROJECT_DELETION_TARGET_CONCURRENCY, default 12) applied within each phase — targets of the same kind now run concurrently, but phases still run strictly in order (runtime, then workspace, then bos, then records), preserving the "stop/verify runtime, then remove files, then DB" ordering the plan is built with. The one-time phase-transition safety checks (wait for in-flight human work, re-verify runtime absence, revalidate references) still run exactly once, at the same point, between the runtime and workspace phases. Every read-modify-write of shared task state (a target's Status/Error, and every persist() call) is guarded by a mutex; the actual I/O (deleteProjectTarget) runs unlocked so workers genuinely overlap. Added TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes (30 runs) to lock in that concurrent execution reaches the same end state — every target done, every workspace directory removed, project record gone — as the old sequential loop. Full suite passes under -race with no warnings; 75 pre-existing unrelated failures unchanged. Co-Authored-By: Claude Sonnet 5 --- internal/api/project_deletion.go | 239 +++++++++++++++++--------- internal/api/project_deletion_test.go | 75 ++++++++ 2 files changed, 229 insertions(+), 85 deletions(-) diff --git a/internal/api/project_deletion.go b/internal/api/project_deletion.go index e31033cc..957d75b4 100644 --- a/internal/api/project_deletion.go +++ b/internal/api/project_deletion.go @@ -18,7 +18,9 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" + "sync" "time" ) @@ -646,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s } } } - runtimeDone := false - for index := range task.Plan.Targets { - target := &task.Plan.Targets[index] - if target.Status == "done" { - continue + targetConcurrency := 12 + if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" { + if n, e := strconv.Atoi(v); e == nil && n > 0 { + targetConcurrency = n } - if target.Kind != "runtime" && !runtimeDone { - deadline := time.Now().Add(150 * time.Second) - for { - items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID}) - if e != nil { - fail(e) - return - } - busy := false - for _, id := range task.Plan.RunIDs { - run, e := s.store.GetRun(ctx, id) - if e != nil { - fail(e) - return - } - if run != nil && run.ExecutionLeaseOwner != "" { - until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt) - if e != nil || until.After(time.Now()) { - busy = true - } - } - } - for i := range items { - item := &items[i] - if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil { - fail(e) - return - } - w := mapValue(item.Subject.Payload["sliceWorkflow"]) - if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" { - busy = true - } - } - if !busy { - break - } - if time.Now().After(deadline) { - fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试")) - return - } - select { - case <-ctx.Done(): - fail(ctx.Err()) - return - case <-time.After(2 * time.Second): - } + } + // processBatch runs deleteProjectTarget for the not-yet-done targets at + // indices with up to targetConcurrency in flight at once. Each target's + // Status/Error and every persist() call is guarded by mu, so concurrent + // workers never race on shared task state; deleteProjectTarget itself + // (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs + // unlocked. A single project's deletion used to process every target + // strictly one at a time, which made stopping thousands of real runs + // take as long as thousands of sequential K8s round trips. + var mu sync.Mutex + processBatch := func(indices []int) error { + sem := make(chan struct{}, targetConcurrency) + var wg sync.WaitGroup + var firstErr error + for _, index := range indices { + mu.Lock() + skip := task.Plan.Targets[index].Status == "done" || firstErr != nil + mu.Unlock() + if skip { + continue } - // Catch work admitted immediately before the project lock. - for _, step := range task.Plan.Targets { - if step.Kind == "runtime" { - if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil { - fail(e) - return - } + sem <- struct{}{} + wg.Add(1) + go func(index int) { + defer wg.Done() + defer func() { <-sem }() + mu.Lock() + if firstErr != nil { + mu.Unlock() + return } - } - // Scoped exactly as before (projectId match only, no workflowId - // fallback — see the loop below) but via an indexed query - // instead of an organization-wide scan. - fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil) + target := &task.Plan.Targets[index] + target.Error = "" + target.Status = "running" + persistErr := persist() + snapshot := *target + mu.Unlock() + if persistErr != nil { + mu.Lock() + if firstErr == nil { + firstErr = persistErr + } + mu.Unlock() + cancel() + return + } + runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot) + mu.Lock() + target = &task.Plan.Targets[index] + if runErr != nil { + target.Status = "failed" + target.Error = runErr.Error() + if firstErr == nil { + firstErr = runErr + } + } else { + target.Status = "done" + } + _ = persist() + mu.Unlock() + }(index) + } + wg.Wait() + return firstErr + } + indicesByKind := map[string][]int{} + nonRuntimePending := false + for index, t := range task.Plan.Targets { + indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index) + if t.Kind != "runtime" && t.Status != "done" { + nonRuntimePending = true + } + } + if e := processBatch(indicesByKind["runtime"]); e != nil { + fail(e) + return + } + // Re-run the phase-transition safety checks (wait for in-flight human + // work, re-verify runtime absence, revalidate references) exactly once + // per execution attempt, right before touching any file/storage target + // — the same point the original sequential loop ran them at. + if nonRuntimePending { + deadline := time.Now().Add(150 * time.Second) + for { + items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID}) if e != nil { fail(e) return } - known := map[string]bool{} + busy := false for _, id := range task.Plan.RunIDs { - known[id] = true + run, e := s.store.GetRun(ctx, id) + if e != nil { + fail(e) + return + } + if run != nil && run.ExecutionLeaseOwner != "" { + until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt) + if e != nil || until.After(time.Now()) { + busy = true + } + } } - for _, r := range fresh { - if r.ProjectID == task.Plan.ProjectID && !known[r.ID] { - fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID)) + for i := range items { + item := &items[i] + if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil { + fail(e) + return + } + w := mapValue(item.Subject.Payload["sliceWorkflow"]) + if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" { + busy = true + } + } + if !busy { + break + } + if time.Now().After(deadline) { + fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试")) + return + } + select { + case <-ctx.Done(): + fail(ctx.Err()) + return + case <-time.After(2 * time.Second): + } + } + // Catch work admitted immediately before the project lock. + for _, step := range task.Plan.Targets { + if step.Kind == "runtime" { + if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil { + fail(e) return } } - if e = s.validateDeletionReferences(ctx, task.Plan); e != nil { - fail(e) - return - } - runtimeDone = true } - target.Error = "" - target.Status = "running" - if e := persist(); e != nil { - cancel() - return - } - if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil { - target.Status = "failed" - target.Error = e.Error() + // Scoped exactly as before (projectId match only, no workflowId + // fallback — see the loop below) but via an indexed query + // instead of an organization-wide scan. + fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil) + if e != nil { fail(e) return } - target.Status = "done" - if e := persist(); e != nil { - cancel() + known := map[string]bool{} + for _, id := range task.Plan.RunIDs { + known[id] = true + } + for _, r := range fresh { + if r.ProjectID == task.Plan.ProjectID && !known[r.ID] { + fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID)) + return + } + } + if e = s.validateDeletionReferences(ctx, task.Plan); e != nil { + fail(e) + return + } + } + for _, kind := range []string{"workspace", "bos", "records"} { + if e := processBatch(indicesByKind[kind]); e != nil { + fail(e) return } } diff --git a/internal/api/project_deletion_test.go b/internal/api/project_deletion_test.go index 62db2d0b..5ad95e9a 100644 --- a/internal/api/project_deletion_test.go +++ b/internal/api/project_deletion_test.go @@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) { t.Fatal("raw data removed") } } + +// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the +// processBatch concurrency added to executeProjectDeletion: with many runs +// (many runtime + workspace targets processed by a worker pool instead of +// one at a time), the task must still end up "completed" with every target +// "done" and every run's workspace directory actually removed — i.e. the +// concurrent rewrite produces the same end state as the old sequential +// loop, just faster. +func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) { + ctx := context.Background() + db := store.NewMemoryStore() + p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"}) + if e != nil { + t.Fatal(e) + } + wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID}) + if e != nil { + t.Fatal(e) + } + s := &server{store: db, workspaceMountPath: t.TempDir()} + const runCount = 30 + dirs := make([]string, 0, runCount) + for i := 0; i < runCount; i++ { + runID := fmt.Sprintf("run-concurrent-fixture-%02d", i) + if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil { + t.Fatal(e) + } + dir, e := safeRunWorkspace(s.workspaceMountPath, runID) + if e != nil { + t.Fatal(e) + } + if e = os.MkdirAll(dir, 0700); e != nil { + t.Fatal(e) + } + dirs = append(dirs, dir) + } + plan, e := s.buildProjectDeletionPlan(ctx, p) + if e != nil { + t.Fatal(e) + } + if len(plan.Blockers) > 0 { + t.Fatal(plan.Blockers) + } + task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"}) + if e != nil { + t.Fatal(e) + } + old := projectDeletionRuntimeFactory + defer func() { projectDeletionRuntimeFactory = old }() + projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) { + return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil + } + claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute))) + if e != nil || !ok { + t.Fatal(e) + } + s.executeProjectDeletion(*claimed, "owner") + finished, _ := db.GetProjectDeletion(ctx, task.ID) + if finished.Status != "completed" { + t.Fatalf("%+v", finished) + } + for _, target := range finished.Plan.Targets { + if target.Status != "done" { + t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target) + } + } + for _, dir := range dirs { + if _, e = os.Stat(dir); !os.IsNotExist(e) { + t.Fatalf("workspace %s not removed", dir) + } + } + if existing, _ := db.GetProject(ctx, p.ID); existing != nil { + t.Fatal("project remains") + } +} From c3edb528d781ef6db2814c06c1d16b6ffbd7f5d6 Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 23 Sep 2026 11:18:24 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=90=9B=20Fix=20account=20creation=20m?= =?UTF-8?q?isdiagnosed=20as=20duplicate=20phone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root-caused live on Fanyu: creating a new account failed with "phone must be unique" for a phone number that had never been used. The users collection still carried a stale, non-sparse email_unique index left over from before accounts moved to phone as the sole identifier — this codebase stopped declaring that index, but nothing ever dropped it from an already-existing database. The first account created without an email field (a normal, expected shape post-migration) occupied that index's one allowed null slot; every account created after that collided with it on insert, and upsertUser/updateAdministrator treated *any* MongoDB duplicate-key error on the users collection as a phone conflict without checking which index actually fired, misreporting the real (email_unique) conflict as a phone conflict that had nothing to do with the phone number entered. Fixes: - isPhoneUniqueDuplicateKeyError checks the error message for the real index name (phone_unique) instead of assuming any duplicate-key error on the collection is a phone conflict — mirrors the existing pattern already used for annotation_labels' unique-index checks. - EnsureIndexes now drops email_unique if present (dropObsoleteIndexes), so any other pre-migration database doesn't carry the same landmine. - Directly dropped the stale email_unique index on the live Fanyu database to unblock account creation immediately. Added TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex (live-Mongo gated, same convention as the existing annotation-review index test) to reproduce the exact failure and lock in the fix. Co-Authored-By: Claude Sonnet 5 --- internal/store/account_admin_guard.go | 2 +- ...ccount_phone_duplicate_error_mongo_test.go | 88 +++++++++++++++++++ internal/store/mongo.go | 11 ++- internal/store/mongo_indexes.go | 23 +++++ 4 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 internal/store/account_phone_duplicate_error_mongo_test.go diff --git a/internal/store/account_admin_guard.go b/internal/store/account_admin_guard.go index fee6a107..d0e79270 100644 --- a/internal/store/account_admin_guard.go +++ b/internal/store/account_admin_guard.go @@ -84,7 +84,7 @@ func (store *MongoStore) updateAdministrator(ctx context.Context, item domain.Us _, e = store.users.UpdateOne(sc, immutableSupplierFilter(item.ID, item.SupplierID), bson.M{"$set": fields}) return nil, e }) - if mongo.IsDuplicateKeyError(err) { + if isPhoneUniqueDuplicateKeyError(err) { return ErrDuplicateUserPhone } return err diff --git a/internal/store/account_phone_duplicate_error_mongo_test.go b/internal/store/account_phone_duplicate_error_mongo_test.go new file mode 100644 index 00000000..921c6565 --- /dev/null +++ b/internal/store/account_phone_duplicate_error_mongo_test.go @@ -0,0 +1,88 @@ +package store + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex reproduces the exact +// bug found live on Fanyu: a users collection that still carries a stale, +// pre-phone-migration email_unique index (this codebase no longer declares +// it, but an older database can still have it — see EnsureIndexes' +// dropObsoleteIndexes call). A duplicate-key error from that unrelated index +// must not be misreported as a phone conflict, or every account created +// after the first one missing an email field fails with a "phone must be +// unique" error that has nothing to do with the phone number entered. +func TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex(t *testing.T) { + uri := os.Getenv("MONGO_INDEX_TEST_URI") + if uri == "" { + t.Skip("MONGO_INDEX_TEST_URI required for real Mongo index regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(ctx) + db := os.Getenv("MONGO_INDEX_TEST_DB") + if db == "" { + t.Fatal("MONGO_INDEX_TEST_DB required") + } + c := client.Database(db).Collection(fmt.Sprintf("users_phone_dup_test_%d", time.Now().UnixNano())) + defer c.Drop(context.Background()) + + // A pre-migration-shaped users collection: phone_unique (current) plus a + // leftover, non-sparse email_unique (obsolete, but not yet dropped). + if _, err := c.Indexes().CreateMany(ctx, []mongo.IndexModel{ + uniqueIndex("phone_unique", bson.D{{Key: "phone", Value: 1}}), + uniqueIndex("email_unique", bson.D{{Key: "email", Value: 1}}), + }); err != nil { + t.Fatal(err) + } + + // First account has no email field — occupies email_unique's one null slot. + if _, err := c.InsertOne(ctx, bson.M{"id": "user-a", "phone": "13800000001"}); err != nil { + t.Fatal(err) + } + + // A second account, also missing email but with a genuinely distinct + // phone, collides only with the obsolete email index — must NOT be + // classified as a phone conflict. + _, err = c.InsertOne(ctx, bson.M{"id": "user-b", "phone": "13800000002"}) + if !mongo.IsDuplicateKeyError(err) { + t.Fatalf("expected a duplicate-key error from the stale email index, got: %v", err) + } + if isPhoneUniqueDuplicateKeyError(err) { + t.Fatalf("email_unique conflict misclassified as a phone conflict: %v", err) + } + + // A third account reusing user-a's actual phone must still be correctly + // classified as a real phone conflict. + _, err = c.InsertOne(ctx, bson.M{"id": "user-c", "phone": "13800000001", "email": "user-c@example.com"}) + if !mongo.IsDuplicateKeyError(err) { + t.Fatalf("expected a duplicate-key error from the phone index, got: %v", err) + } + if !isPhoneUniqueDuplicateKeyError(err) { + t.Fatalf("real phone_unique conflict not recognized: %v", err) + } + + if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil { + t.Fatal(err) + } + if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil { + t.Fatalf("dropping an already-absent index must be a no-op: %v", err) + } + + // With the obsolete index gone, a second account missing email succeeds. + if _, err := c.InsertOne(ctx, bson.M{"id": "user-d", "phone": "13800000003"}); err != nil { + t.Fatalf("account creation still blocked after dropping the obsolete index: %v", err) + } +} diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 6d61f07c..7f7a9c85 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -2222,12 +2222,21 @@ func (store *MongoStore) upsertUser(ctx context.Context, item domain.UserRecord, return ErrUserMutationConflict } } - if mongo.IsDuplicateKeyError(err) { + if isPhoneUniqueDuplicateKeyError(err) { return ErrDuplicateUserPhone } return err } +// isPhoneUniqueDuplicateKeyError reports whether err is a MongoDB duplicate-key +// error specifically from the users collection's phone_unique index, not just +// any unique-index conflict on that collection — a stale or unrelated index +// (e.g. a pre-migration email_unique index left over in an older database) +// would otherwise be misreported to the caller as "phone must be unique". +func isPhoneUniqueDuplicateKeyError(err error) bool { + return mongo.IsDuplicateKeyError(err) && strings.Contains(err.Error(), "phone_unique") +} + func (store *MongoStore) CreateSession(ctx context.Context, item domain.SessionRecord) (*domain.SessionRecord, error) { if strings.TrimSpace(item.ID) == "" { item.ID = domain.NewID("session") diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 72fcdcb4..48df2d47 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -378,5 +379,27 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { return err } } + return dropObsoleteIndexes(ctx, store.users, "email_unique") +} + +// dropObsoleteIndexes removes indexes this codebase no longer declares. A +// database created before the accounts moved from email to phone as the +// sole identifier can still carry a non-sparse email_unique index (this +// function never creates it — only pre-migration deployments have it); left +// in place, the first account created without an email field occupies that +// index's one allowed null slot, and every account created after that +// mislabels as "phone must be unique" since the caller only checks whether +// *some* unique index on the collection conflicted. Missing-index errors +// are ignored since that's the desired end state. +func dropObsoleteIndexes(ctx context.Context, collection *mongo.Collection, names ...string) error { + for _, name := range names { + if _, err := collection.Indexes().DropOne(ctx, name); err != nil { + var cmdErr mongo.CommandError + if errors.As(err, &cmdErr) && cmdErr.Code == 27 { // IndexNotFound + continue + } + return err + } + } return nil } From 21277a1887c15783b0670623aa727037f8cccd87 Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 23 Sep 2026 14:02:34 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E2=9C=A8=20Archive=20old=20terminal=20runs?= =?UTF-8?q?=20to=20bound=20org-wide=20scan=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An organization's run history grows unbounded and nothing ever shrinks it — completed/failed/cancelled runs stay in the runs collection forever, only removed if their whole project is later deleted. That made every org-wide scan (most notably validateDeletionReferences' deletion-safety check, and any other query through ListRunArtifactsByOrganization) get proportionally slower over time as an org's run count kept climbing, independent of the size of whatever was actually being operated on. Live on Fanyu this showed up as project deletion getting slower every day, driven entirely by an unrelated capture-processing backlog continuously adding new runs to the same organization. Adds RunRecord.ArchivedAt and a periodic worker sweep (archiveTerminalRuns, mirroring the existing orphan-runtime reconciliation's throttle pattern) that marks completed/failed/cancelled runs whose EndedAt is older than WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS (default 30 days) as archived, checking at most every WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS (default 1 hour). Archiving never deletes anything — ListRunArtifactsByOrganization now excludes archived runs so the hot-path org-wide scans stay bounded, but ListRunArtifactsForProject (used when a project is actually being deleted) still includes them, since a project's own deletion must still clean up every run it ever had regardless of archive state. A run's data is only actually removed when its project is deleted, unchanged from today. Co-Authored-By: Claude Sonnet 5 --- internal/domain/types.go | 6 ++ internal/store/memory.go | 42 ++++++++++++-- internal/store/memory_test.go | 99 +++++++++++++++++++++++++++++++++ internal/store/mongo.go | 20 ++++++- internal/store/mongo_indexes.go | 3 + internal/store/store.go | 8 +++ internal/worker/runtime.go | 39 +++++++++++++ 7 files changed, 212 insertions(+), 5 deletions(-) diff --git a/internal/domain/types.go b/internal/domain/types.go index 563cddf3..acfdd0d9 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -872,6 +872,12 @@ type RunRecord struct { Error string `json:"error,omitempty" bson:"error,omitempty"` CreatedAt string `json:"createdAt" bson:"createdAt"` UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + // ArchivedAt marks a terminal run excluded from org-wide scans (e.g. the + // deletion-safety reference check) once it's old enough that its data no + // longer needs to participate in day-to-day queries. The record itself is + // untouched — archiving never deletes it; only deleting the run's own + // project does that. + ArchivedAt string `json:"archivedAt,omitempty" bson:"archivedAt,omitempty"` } type RunSummary struct { diff --git a/internal/store/memory.go b/internal/store/memory.go index 0736cf78..6f884ad1 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -1281,23 +1281,34 @@ func (store *MemoryStore) ListRunArtifacts(ctx context.Context, status string) ( return items, nil } +// Excludes archived runs, matching MongoStore's ListRunArtifactsByOrganization +// (see ArchiveTerminalRuns) — this is the org-wide scan behind +// validateDeletionReferences' safety check. func (store *MemoryStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) { items, err := store.ListRunsByOrganization(ctx, organizationID, status) if err != nil { return nil, err } - hydrateRunArtifactsWhenNeeded(items) - return items, nil + kept := make([]domain.RunRecord, 0, len(items)) + for _, run := range items { + if run.ArchivedAt == "" { + kept = append(kept, run) + } + } + hydrateRunArtifactsWhenNeeded(kept) + return kept, nil } // ListRunArtifactsForProject mirrors the Mongo implementation's scope: runs // whose projectId matches projectID directly, or whose workflowId is in -// workflowIDs. +// workflowIDs. Deliberately includes archived runs — a project's own +// deletion must still clean up every run it ever had, archived or not. func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) { - all, err := store.ListRunArtifactsByOrganization(ctx, organizationID, "") + all, err := store.ListRunsByOrganization(ctx, organizationID, "") if err != nil { return nil, err } + hydrateRunArtifactsWhenNeeded(all) workflowSet := make(map[string]bool, len(workflowIDs)) for _, id := range workflowIDs { workflowSet[id] = true @@ -1311,6 +1322,29 @@ func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organi return items, nil } +// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns. +func (store *MemoryStore) ArchiveTerminalRuns(_ context.Context, olderThan time.Time) (int, error) { + store.mu.Lock() + defer store.mu.Unlock() + threshold := olderThan.UTC().Format(time.RFC3339Nano) + archived := 0 + for id, run := range store.runs { + if run.ArchivedAt != "" { + continue + } + if run.Status != domain.RunStatusCompleted && run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled { + continue + } + if run.EndedAt == "" || run.EndedAt >= threshold { + continue + } + run.ArchivedAt = nowISO() + store.runs[id] = run + archived++ + } + return archived, nil +} + func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunStatus]int, error) { store.mu.Lock() defer store.mu.Unlock() diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go index a749637a..3976005e 100644 --- a/internal/store/memory_test.go +++ b/internal/store/memory_test.go @@ -1211,3 +1211,102 @@ func TestListRunArtifactsForProjectScopesByProjectOrWorkflow(t *testing.T) { t.Fatalf("scope leaked unrelated runs: %#v", got) } } + +func TestArchiveTerminalRunsOnlyArchivesOldTerminalUnarchivedRuns(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano) + recent := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339Nano) + threshold := time.Now().Add(-30 * 24 * time.Hour) + + runs := []domain.RunRecord{ + {ID: "run-old-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old}, + {ID: "run-old-failed", OrganizationID: "org-1", Status: domain.RunStatusFailed, EndedAt: old}, + {ID: "run-old-cancelled", OrganizationID: "org-1", Status: domain.RunStatusCancelled, EndedAt: old}, + {ID: "run-recent-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: recent}, + {ID: "run-old-running", OrganizationID: "org-1", Status: domain.RunStatusRunning, EndedAt: ""}, + {ID: "run-already-archived", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old}, + } + for _, run := range runs { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + + archived, err := dataStore.ArchiveTerminalRuns(ctx, threshold) + if err != nil { + t.Fatalf("archive terminal runs: %v", err) + } + if archived != 3 { + t.Fatalf("expected 3 runs archived, got %d", archived) + } + + for _, id := range []string{"run-old-completed", "run-old-failed", "run-old-cancelled"} { + run, err := dataStore.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run %s: %v", id, err) + } + if run.ArchivedAt == "" { + t.Fatalf("expected %s to be archived", id) + } + } + for _, id := range []string{"run-recent-completed", "run-old-running"} { + run, err := dataStore.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run %s: %v", id, err) + } + if run.ArchivedAt != "" { + t.Fatalf("expected %s to remain unarchived, got %s", id, run.ArchivedAt) + } + } + + // Re-running must not re-archive (and must not double count) an already + // archived run or one that just got archived. + archivedAgain, err := dataStore.ArchiveTerminalRuns(ctx, threshold) + if err != nil { + t.Fatalf("archive terminal runs (second pass): %v", err) + } + if archivedAgain != 0 { + t.Fatalf("expected second pass to archive nothing, archived %d", archivedAgain) + } +} + +func TestListRunArtifactsByOrganizationExcludesArchivedRunsButForProjectStillIncludesThem(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano) + runs := []domain.RunRecord{ + {ID: "run-active", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old}, + {ID: "run-archived", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old}, + } + for _, run := range runs { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + + orgItems, err := dataStore.ListRunArtifactsByOrganization(ctx, "org-1", "") + if err != nil { + t.Fatalf("list run artifacts by organization: %v", err) + } + if len(orgItems) != 1 || orgItems[0].ID != "run-active" { + t.Fatalf("expected only the unarchived run from the org-wide scan, got %#v", orgItems) + } + + // A project's own deletion planning must still see its archived runs — + // archiving must never exempt a run from being cleaned up by its own + // project's deletion. + projectItems, err := dataStore.ListRunArtifactsForProject(ctx, "org-1", "project-a", nil) + if err != nil { + t.Fatalf("list run artifacts for project: %v", err) + } + got := map[string]bool{} + for _, item := range projectItems { + got[item.ID] = true + } + if !got["run-active"] || !got["run-archived"] { + t.Fatalf("expected project scope to include both runs regardless of archive state, got %#v", got) + } +} diff --git a/internal/store/mongo.go b/internal/store/mongo.go index dbf28e8e..03179fcc 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -1073,8 +1073,12 @@ func (store *MongoStore) ListRunArtifactsForProject(ctx context.Context, organiz return items, nil } +// Excludes archived runs: this is the query behind validateDeletionReferences' +// org-wide safety scan, and an organization's run history is unbounded and +// never otherwise shrinks, so a run old enough to be archived is excluded +// here to keep that scan's cost from growing forever (see ArchiveTerminalRuns). func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) { - filter := bson.M{} + filter := bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}} if status != "" { filter["status"] = status } @@ -1098,6 +1102,20 @@ func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, org return items, nil } +// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns. +func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) { + filter := bson.M{ + "status": bson.M{"$in": bson.A{string(domain.RunStatusCompleted), string(domain.RunStatusFailed), string(domain.RunStatusCancelled)}}, + "endedAt": bson.M{"$ne": "", "$lt": olderThan.UTC().Format(time.RFC3339Nano)}, + "archivedAt": bson.M{"$in": bson.A{nil, ""}}, + } + result, err := store.runs.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"archivedAt": nowISO()}}) + if err != nil { + return 0, err + } + return int(result.ModifiedCount), nil +} + func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) { cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{ {{Key: "$group", Value: bson.M{ diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 48df2d47..3c64fcd4 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -119,6 +119,9 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { index("status_updatedAt", bson.D{{Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}), index("status_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}), index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}), + // Backs ArchiveTerminalRuns' periodic sweep for terminal, old, + // not-yet-archived runs. + index("status_endedAt", bson.D{{Key: "status", Value: 1}, {Key: "endedAt", Value: 1}}), index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}), index("workflowId_taskId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}), index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}), diff --git a/internal/store/store.go b/internal/store/store.go index a9516f13..1b419fb2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -104,6 +104,14 @@ type Store interface { // every run in the organization (see account-phone-login-era perf fix: // large orgs made the unscoped org-wide scan hang for minutes). ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) + // ArchiveTerminalRuns sets ArchivedAt on every not-yet-archived run whose + // status is completed/failed/cancelled and whose EndedAt is older than + // olderThan. Archiving never deletes the record — it only excludes it + // from the org-wide scans (see ListRunArtifactsByOrganization) that + // would otherwise keep growing as an organization's run history grows + // unbounded; a run's data is only actually removed when its project is + // deleted. Returns the number of runs archived. + ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error) ListRunSummaries(context.Context, string) ([]domain.RunSummary, error) ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error) diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 7e4202fd..49eafe7d 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -99,6 +99,9 @@ type Runtime struct { lastOrphanCheckNano atomic.Int64 pendingFailureAfter time.Duration missingMaxRecoveries int + runArchiveMinimumAge time.Duration + runArchiveInterval time.Duration + lastRunArchiveNano atomic.Int64 } type ProcessOnceStatus string @@ -203,6 +206,8 @@ func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName str orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute), pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute), missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3), + runArchiveMinimumAge: envDurationSeconds("WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS", 30*24*time.Hour), + runArchiveInterval: envDurationSeconds("WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS", time.Hour), }, nil } @@ -1076,6 +1081,39 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) { runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors) } +// archiveTerminalRuns periodically marks old, terminal runs as archived (see +// domain.RunRecord.ArchivedAt) so an organization's ever-growing run history +// doesn't keep inflating the cost of org-wide scans like +// validateDeletionReferences' safety check. Throttled the same way +// reconcileOrphanedRuntimes is, since this only needs to run occasionally, +// not on every poll tick. +func (runtime *Runtime) archiveTerminalRuns(ctx context.Context) { + now := time.Now().UTC() + interval := runtime.runArchiveInterval + if interval <= 0 { + interval = time.Hour + } + lastCheck := runtime.lastRunArchiveNano.Load() + if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval { + return + } + if !runtime.lastRunArchiveNano.CompareAndSwap(lastCheck, now.UnixNano()) { + return + } + minimumAge := runtime.runArchiveMinimumAge + if minimumAge <= 0 { + minimumAge = 30 * 24 * time.Hour + } + archived, err := runtime.store.ArchiveTerminalRuns(ctx, now.Add(-minimumAge)) + if err != nil { + fmt.Printf("[worker] archive terminal runs failed: %v\n", err) + return + } + if archived > 0 { + fmt.Printf("[worker] archived %d terminal run(s) older than %s\n", archived, minimumAge) + } +} + func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { startedAt := time.Now() var processErr error @@ -1138,6 +1176,7 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { } runtime.applyWorkflowRuntimePolicies(ctx, workflows) runtime.reconcileOrphanedRuntimes(ctx) + runtime.archiveTerminalRuns(ctx) queuedRuns := runRecordsFromSummaries(queuedPage.Items) selectedRunIDs := selectQueuedRunsForDispatchSnapshot( queuedRuns, From b9ae0ba6531a680344ebf3830da20166a7f144f5 Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 23 Sep 2026 14:38:00 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E2=9A=A1=20Add=20an=20expiring,=20renewed?= =?UTF-8?q?=20lease=20to=20the=20account=20mutation=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-resource mutex behind account/project-member/supplier writes had no expiring lease by design: a paused process must never resume after another writer has stolen its lock, so recovery required an operator to verify the owner had actually stopped and clear the lock by hand. In practice, on Fanyu this meant every worker pod replacement (a routine rolling deploy) mid-hold left the lock permanently stuck — hit three times in one session, each requiring a manual mongosh intervention. Adds a lease (ACCOUNT_MUTATION_LOCK_LEASE_SECONDS, default 2 minutes) renewed by a background ticker for as long as the guarded operation keeps running — mirroring the project-deletion task lease's own renewal pattern already in this codebase. A live owner's lease is renewed well before it can lapse (every leaseDuration/4), so it's never reclaimed out from under it; only a lock whose owner stopped renewing entirely — because it crashed or was replaced — becomes reclaimable, once its lease lapses. Reclaiming always assigns a fresh owner id, so the original holder's own release (matched on "owner": owner) can never succeed afterward and can never touch the new holder's lock — the fresh id already acts as the fencing token a naive TTL-lock would be missing. A transient renewal error (network blip) doesn't give up the lease early — only a confirmed "someone else now owns this" does. Genuine crashes or ambiguous network results are still surfaced to the caller to verify, exactly as before; only the "must wait for an operator to notice and clear it by hand" half of the old design is gone. Added TestAccountMutationLockLeaseReclaimAndRenewal (live-Mongo gated, same convention as the existing index-migration/duplicate-key tests): normal acquire/release still round-trips, a lapsed lease from a simulated dead owner is reclaimable, and a live owner's renewed lease blocks a concurrent reclaim attempt. Co-Authored-By: Claude Sonnet 5 --- internal/store/account_mutation_lock.go | 92 ++++++++++- .../store/account_mutation_lock_mongo_test.go | 145 ++++++++++++++++++ 2 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 internal/store/account_mutation_lock_mongo_test.go diff --git a/internal/store/account_mutation_lock.go b/internal/store/account_mutation_lock.go index 4b812480..72f22993 100644 --- a/internal/store/account_mutation_lock.go +++ b/internal/store/account_mutation_lock.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "os" + "strconv" + "strings" "time" "droboticflow/internal/domain" @@ -14,22 +16,60 @@ import ( var ErrAccountMutationBusy = errors.New("account mutation is busy or requires recovery") -// A durable, per-resource mutex also works on standalone MongoDB. It deliberately -// has no expiring lease: a paused process must never resume after another writer -// has stolen its lock. After a process crash or ambiguous network result, an -// operator must verify that owner has stopped and inspect the affected resource -// before clearing that exact owner's lock. Normal exits release their own lock. +// accountMutationLockLeaseDuration bounds how long a held lock survives its +// owner going silent (crash, pod replacement, OOM) before another process +// may reclaim it. Renewed well before expiry by a background ticker while +// the guarded operation is still running (see withAccountMutationLock), so +// a live owner's lease never lapses out from under it — only a dead one's +// does. Configurable because "how long is safe to wait before auto- +// reclaiming" is an operational judgment call, not a constant. +func accountMutationLockLeaseDuration() time.Duration { + const fallback = 2 * time.Minute + raw := strings.TrimSpace(os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS")) + if raw == "" { + return fallback + } + seconds, err := strconv.Atoi(raw) + if err != nil || seconds <= 0 { + return fallback + } + return time.Duration(seconds) * time.Second +} + +// A durable, per-resource mutex also works on standalone MongoDB. It carries +// an expiring lease (accountMutationLockLeaseDuration) renewed periodically +// by a background ticker for as long as the guarded operation keeps running, +// so a live owner's lock is never reclaimed out from under it — only a lock +// whose owner stopped renewing (crashed, was replaced, or otherwise went +// silent) becomes reclaimable, once its lease lapses. Reclaiming a lapsed +// lease always assigns a fresh owner id, so the original holder's own +// release (guarded by "owner": owner) can never match afterward, and never +// touches the new holder's lock — the fresh id already acts as the fencing +// token a naive TTL-lock would be missing. A crash or ambiguous network +// result during the guarded write is still surfaced for the caller to +// verify, exactly as before; only the "must wait for an operator to notice +// and clear it by hand" half of the old design is gone. func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, operation func(context.Context) (any, error)) (result any, err error) { col := s.db.Collection("accountMutationLocks") _, e := col.InsertOne(ctx, bson.M{"_id": key, "locked": false}) if e != nil && !mongo.IsDuplicateKeyError(e) { return nil, e } + leaseDuration := accountMutationLockLeaseDuration() owner := domain.NewID("mutation") waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() for { - r, e := col.UpdateOne(waitCtx, bson.M{"_id": key, "locked": false}, bson.M{"$set": bson.M{"locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"), "startedAt": time.Now().UTC()}}) + now := time.Now().UTC() + filter := bson.M{"_id": key, "$or": bson.A{ + bson.M{"locked": false}, + bson.M{"locked": true, "leaseExpiresAt": bson.M{"$lt": now}}, + }} + update := bson.M{"$set": bson.M{ + "locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"), + "startedAt": now, "leaseExpiresAt": now.Add(leaseDuration), + }} + r, e := col.UpdateOne(waitCtx, filter, update) if e != nil { return nil, e } @@ -42,14 +82,52 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op case <-time.After(25 * time.Millisecond): } } + // Renew well inside the lease window (a fixed fraction of it) so normal + // scheduling jitter never lets a live owner's lease lapse; only an + // owner that stops renewing entirely — because it's gone — loses the + // lock. Mirrors the project-deletion task lease's own renewal ticker. + renewCtx, cancelRenew := context.WithCancel(context.Background()) + renewalDone := make(chan struct{}) + go func() { + defer close(renewalDone) + ticker := time.NewTicker(leaseDuration / 4) + defer ticker.Stop() + for { + select { + case <-renewCtx.Done(): + return + case <-ticker.C: + renewalCtx, renewalCancel := context.WithTimeout(context.Background(), 5*time.Second) + r, e := col.UpdateOne(renewalCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"leaseExpiresAt": time.Now().UTC().Add(leaseDuration)}}) + renewalCancel() + if e != nil { + // A transient error (network blip, timeout) — not proof + // the lock was actually lost. Keep the lease as-is and + // retry on the next tick; giving up here on a live owner + // would reintroduce the exact "two concurrent writers" + // race the lease is meant to prevent. + continue + } + if r.MatchedCount != 1 { + // The document no longer matches our owner: someone else + // genuinely reclaimed it after our lease lapsed. Stop + // renewing; the release below will correctly no-op or + // surface the conflict. + return + } + } + } + }() defer func() { + cancelRenew() + <-renewalDone // Do not unlock an uncertain write and admit a conflicting operation. if ctx.Err() != nil || mongo.IsNetworkError(err) || mongo.IsTimeout(err) { return } releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second) defer releaseCancel() - r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": ""}}) + r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": "", "leaseExpiresAt": ""}}) if e != nil || r.MatchedCount != 1 { err = fmt.Errorf("%w: operation finished but lock release needs verification", ErrAccountMutationBusy) } diff --git a/internal/store/account_mutation_lock_mongo_test.go b/internal/store/account_mutation_lock_mongo_test.go new file mode 100644 index 00000000..8c039372 --- /dev/null +++ b/internal/store/account_mutation_lock_mongo_test.go @@ -0,0 +1,145 @@ +package store + +import ( + "context" + "errors" + "os" + "sync/atomic" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// This test deliberately uses MongoDB, not MemoryStore: the lease/reclaim +// behavior is a persisted-document race the in-memory store can't +// reproduce. Found live on Fanyu: a lock whose owning process was replaced +// mid-hold (a rolling worker deploy, or a crash) stayed "locked" forever — +// the original design had no expiring lease by intent, trading availability +// for a hand-verified recovery. This locks in the replacement: an expiring, +// renewed lease that only a genuinely dead owner ever loses. +func TestAccountMutationLockLeaseReclaimAndRenewal(t *testing.T) { + uri := os.Getenv("MONGO_INDEX_TEST_URI") + if uri == "" { + t.Skip("MONGO_INDEX_TEST_URI required for real Mongo lock regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(ctx) + db := os.Getenv("MONGO_INDEX_TEST_DB") + if db == "" { + t.Fatal("MONGO_INDEX_TEST_DB required") + } + dbHandle := client.Database(db) + defer dbHandle.Collection("accountMutationLocks").Drop(context.Background()) + s := &MongoStore{db: dbHandle} + + t.Run("normal acquire and release round-trips and unblocks the next waiter", func(t *testing.T) { + key := "lease-test-normal" + _, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "first", nil }) + if err != nil { + t.Fatalf("first acquire: %v", err) + } + var doc bson.M + if err := dbHandle.Collection("accountMutationLocks").FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil { + t.Fatal(err) + } + if doc["locked"] != false { + t.Fatalf("expected lock released after normal completion, got %#v", doc) + } + if _, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "second", nil }); err != nil { + t.Fatalf("second acquire after release: %v", err) + } + }) + + t.Run("a lease that lapsed because its owner went silent is reclaimable", func(t *testing.T) { + key := "lease-test-lapsed" + col := dbHandle.Collection("accountMutationLocks") + // Simulate a crashed holder: locked, with a lease already in the past. + if _, err := col.InsertOne(ctx, bson.M{ + "_id": key, "locked": true, "owner": "mutation-dead-owner", "process": "dead-pod", + "startedAt": time.Now().UTC().Add(-time.Hour), "leaseExpiresAt": time.Now().UTC().Add(-time.Minute), + }); err != nil { + t.Fatal(err) + } + claimed := make(chan struct{}) + result, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { + close(claimed) + return "reclaimed", nil + }) + select { + case <-claimed: + default: + t.Fatal("operation never ran — lapsed lease was not reclaimed") + } + if err != nil { + t.Fatalf("reclaim after lapse: %v", err) + } + if result != "reclaimed" { + t.Fatalf("unexpected result: %v", result) + } + var doc bson.M + if err := col.FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil { + t.Fatal(err) + } + if doc["locked"] != false { + t.Fatalf("expected lock released after reclaim completed, got %#v", doc) + } + }) + + t.Run("a live owner's lease is renewed before it can lapse, blocking a concurrent reclaim attempt", func(t *testing.T) { + key := "lease-test-renewed" + originalLease := os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS") + os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", "1") // renew ticks every ~250ms + defer func() { + if originalLease == "" { + os.Unsetenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS") + } else { + os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", originalLease) + } + }() + + holderStarted := make(chan struct{}) + holderMayFinish := make(chan struct{}) + var reclaimAttempted atomic.Bool + var reclaimErr error + + done := make(chan struct{}) + go func() { + defer close(done) + _, e := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { + close(holderStarted) + <-holderMayFinish + return "held", nil + }) + if e != nil { + t.Errorf("long-held operation returned an error: %v", e) + } + }() + + <-holderStarted + // Outlive several renewal ticks (lease=1s, renew every ~250ms) while + // still holding the lock, then attempt a short-timeout reclaim: it + // must fail, proving the live holder's lease never lapsed. + time.Sleep(2500 * time.Millisecond) + reclaimCtx, reclaimCancel := context.WithTimeout(ctx, 500*time.Millisecond) + _, reclaimErr = s.withAccountMutationLock(reclaimCtx, key, func(context.Context) (any, error) { return "stolen", nil }) + reclaimCancel() + reclaimAttempted.Store(true) + close(holderMayFinish) + <-done + + if !reclaimAttempted.Load() { + t.Fatal("reclaim was never attempted") + } + if !errors.Is(reclaimErr, ErrAccountMutationBusy) { + t.Fatalf("expected the renewed lease to block a concurrent reclaim, got: %v", reclaimErr) + } + }) +} From 1a7eb3bfb3561960789d79028cc7d75351a75abb Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 24 Sep 2026 12:43:29 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=A9=B9=20Keep=20Fanyu=20slice=20proce?= =?UTF-8?q?ssing=20in=20a=20bounded=20external=20worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- k8s/annotation-processor.template.yaml | 6 ++++-- scripts/deploy-envs/fanyu.env | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/k8s/annotation-processor.template.yaml b/k8s/annotation-processor.template.yaml index 76ff931d..c34493e2 100644 --- a/k8s/annotation-processor.template.yaml +++ b/k8s/annotation-processor.template.yaml @@ -6,6 +6,8 @@ metadata: namespace: __NAMESPACE__ spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: {app: droboticflow-annotation-processor} template: @@ -22,8 +24,8 @@ spec: command: [/usr/local/bin/droboticflow-api] env: - {name: ANNOTATION_PROCESSOR_MODE, value: processor} - - {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "2"} - - {name: ANNOTATION_INDEX_BACKFILL, value: "1"} + - {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "1"} + - {name: ANNOTATION_INDEX_BACKFILL, value: "0"} - {name: WORKSPACE_MOUNT_PATH, value: "__WORKSPACE_MOUNT_PATH__"} - {name: GLOBAL_BOS_ENDPOINT, value: "__BOS_ENDPOINT__"} - {name: ANNOTATION_WORKSPACE_CONNECTION_ID, value: "__ANNOTATION_WORKSPACE_CONNECTION_ID__"} diff --git a/scripts/deploy-envs/fanyu.env b/scripts/deploy-envs/fanyu.env index 1888073f..7626a216 100644 --- a/scripts/deploy-envs/fanyu.env +++ b/scripts/deploy-envs/fanyu.env @@ -12,3 +12,6 @@ export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-re export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}" export WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}" export WORKSPACE_BOS_BUCKET="${WORKSPACE_BOS_BUCKET:-drobodata-fanyu}" + +# Keep heavy slice materialization out of the web API. +export ANNOTATION_PROCESSOR_ENABLED="${ANNOTATION_PROCESSOR_ENABLED:-true}" From ce133c0fa88454a71203bedaef3395a1b25bf568 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 24 Sep 2026 15:16:52 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=A9=B9=20Preserve=20action=20records?= =?UTF-8?q?=20when=20consolidating=20invalid-only=20slices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../annotation-invalid-slice-consolidation.md | 13 +++ .../api/annotation_blank_delivery_test.go | 43 +++++++++- internal/api/annotation_current.go | 5 ++ .../annotation_invalid_slice_consolidation.go | 82 +++++++++++++++++++ ...tation_invalid_slice_consolidation_test.go | 82 +++++++++++++++++++ internal/api/annotation_operations.go | 33 +++++++- src/lib/invalid-slice-consolidation.js | 44 ++++++++++ src/pages/AnnotationOperations.tsx | 21 +++++ src/services/api/annotation-operations.ts | 1 + .../invalid-slice-consolidation.test.mjs | 40 +++++++++ 10 files changed, 358 insertions(+), 6 deletions(-) create mode 100644 docs/annotation-invalid-slice-consolidation.md create mode 100644 internal/api/annotation_invalid_slice_consolidation.go create mode 100644 internal/api/annotation_invalid_slice_consolidation_test.go create mode 100644 src/lib/invalid-slice-consolidation.js create mode 100644 tests/frontend/invalid-slice-consolidation.test.mjs diff --git a/docs/annotation-invalid-slice-consolidation.md b/docs/annotation-invalid-slice-consolidation.md new file mode 100644 index 00000000..a43ea22e --- /dev/null +++ b/docs/annotation-invalid-slice-consolidation.md @@ -0,0 +1,13 @@ +# 无效动作与外层切片的无损整理 + +无效动作属于外层业务切片内部,设置无效不新增外层边界。旧版取消切分仍按既有规则清空合并区间判断;它不是无损整理入口。 + +送检前识别以下候选:连续三个保留切片名称完全相同、首尾精确相接,中间切片由明确无效的动作完整覆盖。用户逐项选择是否合并,最后再次确认提交。取消选择可保留原划分,不对所有同名切片做自动合并。舍弃、名称不同、缺口、部分无效都不匹配。 + +服务器重新验证选择,仅在已有领取权、版本、字典、覆盖与动作合法性校验全部通过后,改变送检版本的外层划分。动作数组不因合并改变:ID、描述、无效判断、起止边界、硬件时间戳均保留。重叠候选合成同一连续区间。操作历史记录原切片快照。 + +后台生成新的媒体和 JSON;不能把旧短片直接冒充合并后视频。JSON 中相对于切片起点的时间随新的切片起点正常重基,源视频时间、硬件时间戳、动作含义保持一致。旧版本文件和记录通过任务级历史保留标记保护,后续送检和重切也不会由普通版本清理删除。 + +部署无数据库迁移,不自动修改已送检、已审核任务,也不代替客户送检。现有在审任务需要正常退回至标注后使用此入口;无需重新填写动作。更新现有在审结果并保持原审核状态,需要独立的、明确授权的数据修订流程,不属于本次部署。 + +验证覆盖:候选识别、非法选择拒绝且不占用任务、重叠候选、真实持久队列与媒体生成、导出 JSON 动作与时间戳保留,以及原版本所有文件逐字节未变。 diff --git a/internal/api/annotation_blank_delivery_test.go b/internal/api/annotation_blank_delivery_test.go index db06ad36..2c726104 100644 --- a/internal/api/annotation_blank_delivery_test.go +++ b/internal/api/annotation_blank_delivery_test.go @@ -47,7 +47,7 @@ func TestContinuousBlankActionsReachReviewAndPublishedJSON(t *testing.T) { } } -func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation) { +func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation, consolidate ...bool) { t.Helper() t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") f, root := annotationProcessorFixture(t) @@ -103,6 +103,11 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot } } slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: 4}} + var selected []string + if len(consolidate) > 0 && consolidate[0] { + slices = []domain.SliceSegment{{ID: "left", Name: "测试切片", Start: 0, End: 1}, {ID: "invalid", Name: "测试切片", Start: 1, End: 2}, {ID: "right", Name: "测试切片", Start: 2, End: 4}} + selected = []string{"invalid"} + } version := f.item.Version if reuse { response := f.request(http.MethodPost, endpoint+"/slice-confirmations", f.annotator, map[string]any{"version": version, "sliceSegments": slices}) @@ -111,14 +116,40 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot } version = waitFor(false) } + priorFiles := map[string]string{} + if reuse && len(selected) > 0 { + priorRoot := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, "revision-000001") + err := filepath.Walk(priorRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + data, e := os.ReadFile(path) + if e != nil { + return e + } + priorFiles[path] = string(data) + } + return nil + }) + if err != nil || len(priorFiles) == 0 { + t.Fatalf("prior evidence unavailable: %v", err) + } + } response := f.request(http.MethodPost, endpoint+"/combined-submissions", f.annotator, map[string]any{ "version": version, "requireSliceCompletion": true, - "sliceSegments": slices, "segments": segments, + "sliceSegments": slices, "segments": segments, "consolidateInvalidSliceIds": selected, }) if response.Code != http.StatusAccepted { t.Fatalf("blank submission: %d %s", response.Code, response.Body.String()) } waitFor(true) + for path, original := range priorFiles { + data, e := os.ReadFile(path) + if e != nil || string(data) != original { + t.Fatalf("old revision overwritten: %s %v", path, e) + } + } response = f.request(http.MethodGet, endpoint, f.reviewer, nil) if response.Code != http.StatusOK { t.Fatalf("quality read: %d %s", response.Code, response.Body.String()) @@ -149,7 +180,7 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot if reuse { expectedGeneration = 2 } - if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != reuse { + if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != (reuse && len(selected) == 0) { t.Fatalf("expected generation %d reuse %v: %s", expectedGeneration, reuse, response.Body.String()) } path := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, fmt.Sprintf("revision-%06d", generation), "slices", "slice-000001", "annotation.json") @@ -180,3 +211,9 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot } } } + +func TestInvalidSliceConsolidationRebuildsMediaAndPreservesPublishedActions(t *testing.T) { + actions := []domain.AnnotationSegment{{ID: "left-action", Start: 0, End: 1, Status: "described", Description: "拿起"}, {ID: "invalid-action", Start: 1, End: 2, Status: "invalid", IsInvalid: true, Description: "无效片段"}, {ID: "right-action", Start: 2, End: 4, Status: "described", Description: "放下"}} + want := []blankActionExpectation{{"left-action", "described", "拿起", false, 0, 1, 1000000, 2000000}, {"invalid-action", "invalid", "无效片段", true, 1, 2, 2000000, 3000000}, {"right-action", "described", "放下", false, 2, 4, 3000000, 5000000}} + checkBlankActionsDelivery(t, true, actions, want, true) +} diff --git a/internal/api/annotation_current.go b/internal/api/annotation_current.go index 157bee5e..94150540 100644 --- a/internal/api/annotation_current.go +++ b/internal/api/annotation_current.go @@ -23,6 +23,11 @@ func (s *server) retainCurrentAnnotationData(ctx context.Context, task domain.An if item.LegacyTaskID != task.ID || item.OrganizationID != task.OrganizationID || item.RunID != task.RunID { return fmt.Errorf("annotation retention scope mismatch") } + // A lossless consolidation pins its original evidence for future reslicing + // and submissions, not just the first post-consolidation cleanup. + if item.Subject.Payload["preserveAnnotationRevisionHistory"] == true { + return nil + } // Disposition history is review evidence. Keep its private prior generations; // this does not add selectable delivery versions or alter project deletion. if history, ok := item.Subject.Payload["sliceDispositionHistory"]; ok && history != nil { diff --git a/internal/api/annotation_invalid_slice_consolidation.go b/internal/api/annotation_invalid_slice_consolidation.go new file mode 100644 index 00000000..ae8ec142 --- /dev/null +++ b/internal/api/annotation_invalid_slice_consolidation.go @@ -0,0 +1,82 @@ +package api + +import ( + "fmt" + "sort" + "strings" + + "droboticflow/internal/domain" +) + +// Consolidation is an explicit submission choice, never a migration or a +// cancel-cut. Action records are not passed by pointer or rewritten here. +func consolidateInvalidActionSlices(input []domain.SliceSegment, actions []domain.AnnotationSegment, selected []string) ([]domain.SliceSegment, []domain.SliceOperation, error) { + if len(selected) == 0 { + return input, nil, nil + } + ordered := append([]domain.SliceSegment(nil), input...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].Start < ordered[j].Start }) + eligible := map[string]int{} + for i := 1; i < len(ordered)-1; i++ { + l, c, r := ordered[i-1], ordered[i], ordered[i+1] + if l.IsDiscarded() || c.IsDiscarded() || r.IsDiscarded() || strings.TrimSpace(c.Name) == "" || l.Name != c.Name || r.Name != c.Name || l.End != c.Start || c.End != r.Start { + continue + } + cursor, has, valid := c.Start, false, true + inner := append([]domain.AnnotationSegment(nil), actions...) + sort.Slice(inner, func(i, j int) bool { return inner[i].Start < inner[j].Start }) + for _, a := range inner { + if a.End <= c.Start || a.Start >= c.End { + continue + } + if a.Start != cursor || a.End > c.End || a.End <= a.Start || !a.IsInvalid { + valid = false + break + } + cursor, has = a.End, true + } + if valid && has && cursor == c.End { + eligible[c.ID] = i + } + } + edges := make([]bool, len(ordered)) + seen := map[string]bool{} + for _, id := range selected { + i, ok := eligible[id] + if !ok || seen[id] { + return nil, nil, fmt.Errorf("所选无效区间的相邻切片已变化,请重新核对后提交") + } + seen[id] = true + edges[i-1], edges[i] = true, true + } + var result []domain.SliceSegment + var operations []domain.SliceOperation + for i := 0; i < len(ordered); { + j := i + for j < len(ordered)-1 && edges[j] { + j++ + } + merged := ordered[i] + if j > i { + // Keep original IDs in audit provenance; reuse the leftmost stable ID. + merged.SourceIDs = nil + provenance := map[string]bool{} + ids := []string{} + for k := i; k <= j; k++ { + ids = append(ids, ordered[k].ID) + for _, id := range append([]string{ordered[k].ID}, ordered[k].SourceIDs...) { + if !provenance[id] { + merged.SourceIDs = append(merged.SourceIDs, id) + provenance[id] = true + } + } + } + merged.End = ordered[j].End + merged.EndTimestampUS = ordered[j].EndTimestampUS + operations = append(operations, domain.SliceOperation{Type: "consolidate_invalid_actions", SegmentIDs: ids, Before: append([]domain.SliceSegment(nil), ordered[i:j+1]...), After: []domain.SliceSegment{merged}}) + } + result = append(result, merged) + i = j + 1 + } + return result, operations, nil +} diff --git a/internal/api/annotation_invalid_slice_consolidation_test.go b/internal/api/annotation_invalid_slice_consolidation_test.go new file mode 100644 index 00000000..b668e7b8 --- /dev/null +++ b/internal/api/annotation_invalid_slice_consolidation_test.go @@ -0,0 +1,82 @@ +package api + +import ( + "encoding/json" + "reflect" + "testing" + + "droboticflow/internal/domain" +) + +func TestInvalidSliceConsolidationPreservesActionsAndAudit(t *testing.T) { + slices := []domain.SliceSegment{{ID: "a", Name: "收银", Start: 0, End: 2, StartTimestampUS: 100}, {ID: "b", Name: "收银", Start: 2, End: 3}, {ID: "c", Name: "收银", Start: 3, End: 5, EndTimestampUS: 900}, {ID: "d", Name: "其他", Start: 5, End: 9}} + actions := []domain.AnnotationSegment{{ID: "x", Start: 0, End: 2, Description: "拿起商品", Phrases: []string{"细节"}}, {ID: "y", Start: 2, End: 3, IsInvalid: true, Description: "无效片段", Status: "invalid", StartTimestampUS: 200, EndTimestampUS: 300}, {ID: "z", Start: 3, End: 5, Description: "放下商品"}} + before, _ := json.Marshal(actions) + original, _ := json.Marshal(slices) + got, ops, err := consolidateInvalidActionSlices(slices, actions, []string{"b"}) + if err != nil || len(got) != 2 || got[0].Start != 0 || got[0].End != 5 || got[0].Name != "收银" || got[0].StartTimestampUS != 100 || got[0].EndTimestampUS != 900 { + t.Fatalf("%+v %v", got, err) + } + after, _ := json.Marshal(actions) + if string(before) != string(after) { + t.Fatal("action JSON changed") + } + unchanged, _ := json.Marshal(slices) + if string(original) != string(unchanged) { + t.Fatal("input slices mutated") + } + if !reflect.DeepEqual(ops[0].Before, slices[:3]) || ops[0].Type != "consolidate_invalid_actions" { + t.Fatal("audit lost") + } + if !reflect.DeepEqual(got[1], slices[3]) { + t.Fatal("unrelated slice changed") + } + kept, ops, err := consolidateInvalidActionSlices(slices, actions, nil) + if err != nil || len(ops) != 0 || !reflect.DeepEqual(kept, slices) { + t.Fatal("implicit consolidation") + } + for name, change := range map[string]func([]domain.SliceSegment, []domain.AnnotationSegment){ + "discard": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[1].DeliveryDisposition = "discard" }, + "different-name": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Name = "盘点" }, + "gap": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Start = 3.1 }, + "valid-action": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].IsInvalid = false }, + "partial-invalid": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].End = 2.5 }, + } { + t.Run(name, func(t *testing.T) { + s := append([]domain.SliceSegment(nil), slices...) + a := append([]domain.AnnotationSegment(nil), actions...) + change(s, a) + if _, _, err := consolidateInvalidActionSlices(s, a, []string{"b"}); err == nil { + t.Fatal("unsafe consolidation accepted") + } + }) + } + if _, _, err = consolidateInvalidActionSlices(slices, actions, []string{"b", "b"}); err == nil { + t.Fatal("duplicate accepted") + } +} + +func TestInvalidSliceConsolidationOverlappingCandidates(t *testing.T) { + var ss []domain.SliceSegment + var aa []domain.AnnotationSegment + for i, id := range []string{"a", "b", "c", "d", "e"} { + ss = append(ss, domain.SliceSegment{ID: id, Name: "盘点", Start: float64(i), End: float64(i + 1)}) + aa = append(aa, domain.AnnotationSegment{ID: id, Start: float64(i), End: float64(i + 1), IsInvalid: i == 1 || i == 3}) + } + got, ops, err := consolidateInvalidActionSlices(ss, aa, []string{"b", "d"}) + if err != nil || len(got) != 1 || got[0].End != 5 || len(ops) != 1 || len(ops[0].Before) != 5 { + t.Fatalf("%+v %v", got, err) + } +} + +func TestInvalidSliceConsolidationRejectsStaleSelectionBeforeReservation(t *testing.T) { + f := continuousFixture(t) + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": []domain.SliceSegment{{ID: "s", Name: "测试切片", Start: 0, End: 10}}, "segments": []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Description: "动作"}}, "consolidateInvalidSliceIds": []string{"missing"}}) + if r.Code != 400 { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + if item.Version != f.item.Version || item.PendingCombinedSubmissionID != "" { + t.Fatal("invalid request mutated task") + } +} diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index 3d507e9b..f9cd9915 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -1055,6 +1055,7 @@ type sliceRevisionInput struct { // Slice correction and semantic labels are persisted together only after both // validate, so a quality reviewer never receives one without the other. type combinedAnnotationSubmissionInput struct { + ConsolidateInvalidSliceIDs []string `json:"consolidateInvalidSliceIds,omitempty"` // Set only by the internal cut job; never accepted from JSON. sliceOnly bool Version int `json:"version"` @@ -2056,6 +2057,22 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri renderValidationError(writer, issues...) return } + // Original slices, names, disposition and actions have all been validated. + // Only an explicit user selection may change the outer partition. Retain + // action IDs, descriptions, invalid flags and absolute timestamp boundaries. + if len(payload.ConsolidateInvalidSliceIDs) > 0 { + merged, operations, mergeErr := consolidateInvalidActionSlices(payload.SliceSegments, payload.Segments, payload.ConsolidateInvalidSliceIDs) + if mergeErr != nil { + renderValidationError(writer, validationIssue{Field: "consolidateInvalidSliceIds", Message: mergeErr.Error()}) + return + } + if issues := validateDescriptionIntervals(merged, payload.Segments, false); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + payload.SliceSegments = merged + payload.SliceOperations = append(payload.SliceOperations, operations...) + } payloadSHA, hashErr := fanyuV5CombinedPayloadSHA(payload) if hashErr != nil { renderError(writer, http.StatusInternalServerError, hashErr) @@ -2201,6 +2218,9 @@ func (server *server) completeCombinedAnnotationSubmission(writer http.ResponseW if humanItem.Subject.Payload == nil { humanItem.Subject.Payload = map[string]any{} } + if len(payload.ConsolidateInvalidSliceIDs) > 0 { + humanItem.Subject.Payload["preserveAnnotationRevisionHistory"] = true + } humanItem.Subject.Payload["materialization"] = materialization.Payload if workflow := mapValue(humanItem.Subject.Payload["sliceWorkflow"]); len(workflow) > 0 { workflow["materialization"] = materialization.Payload @@ -2999,7 +3019,10 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri return true } } - if operationFromRequest(request) != nil { server.planQualityReviewDecision(writer, request, *task, *item, batch, payload); return true } + if operationFromRequest(request) != nil { + server.planQualityReviewDecision(writer, request, *task, *item, batch, payload) + return true + } now := nowISO() review := domain.AnnotationReviewRecord{ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: task.CurrentRevision, Stage: domain.AnnotationReviewStageQuality, Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, ProblemSliceIDs: payload.ProblemSliceIDs, CreatedAt: now} review.ReviewRound = item.ReviewRound @@ -3131,14 +3154,18 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r renderJSON(writer, http.StatusForbidden, map[string]string{"error": string(requiredRole) + " assignment is required"}) return } - if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) { return } + if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) { + return + } humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "") if !claimed { return } var prepared bool request, prepared = server.prepareReviewDecision(writer, request, *humanItem, "quality", payload.ReviewDecisionInput, payload, string(payload.Decision)) - if !prepared { return } + if !prepared { + return + } defer server.finishReviewDecisionPreparation(request) finishReviewWrite, writeAllowed := server.beginReviewWrite(writer, request, *humanItem) if !writeAllowed { diff --git a/src/lib/invalid-slice-consolidation.js b/src/lib/invalid-slice-consolidation.js new file mode 100644 index 00000000..0651dab3 --- /dev/null +++ b/src/lib/invalid-slice-consolidation.js @@ -0,0 +1,44 @@ +// Candidate discovery only. The server revalidates explicit selections and +// creates a new revision; viewing or marking an action invalid never merges. +export function invalidSliceConsolidationCandidates(slices, actions) { + const ordered = [...slices].sort((a, b) => a.start - b.start); + return ordered.flatMap((current, index) => { + const left = ordered[index - 1]; + const right = ordered[index + 1]; + if ( + !left || + !right || + [left, current, right].some((s) => s.deliveryDisposition === "discard") || + !current.name?.trim() || + left.name !== current.name || + right.name !== current.name || + left.end !== current.start || + current.end !== right.start + ) + return []; + const inner = actions + .filter((a) => a.end > current.start && a.start < current.end) + .sort((a, b) => a.start - b.start); + let cursor = current.start; + if ( + !inner.length || + !inner.every((a) => { + if (!a.isInvalid || a.start !== cursor || a.end <= a.start || a.end > current.end) return false; + cursor = a.end; + return true; + }) || + cursor !== current.end + ) + return []; + return [ + { + id: current.id, + name: current.name, + start: left.start, + end: right.end, + invalidStart: current.start, + invalidEnd: current.end, + }, + ]; + }); +} diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 531044ab..9ab9fa99 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -1,3 +1,4 @@ +import { invalidSliceConsolidationCandidates } from "../lib/invalid-slice-consolidation.js"; import { ReviewDecisionDialog } from "../components/ReviewDecisionDialog"; import { useReviewContinuation } from "../hooks/useReviewContinuation"; import { useReviewEntryIntent } from "../hooks/useReviewEntryIntent"; @@ -3157,12 +3158,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: setError("请保存所有动作描述或明确设为无效"); return; } + const candidates = invalidSliceConsolidationCandidates(sliceSegments, segments); + const consolidateInvalidSliceIds: string[] = []; + for (const candidate of candidates) { + if ( + window.confirm( + `“${candidate.name}”中 ${candidate.invalidStart.toFixed(3)}–${candidate.invalidEnd.toFixed(3)} 秒的无效动作单独占了一个切片。\n\n是否与前后同名切片合并为 ${candidate.start.toFixed(3)}–${candidate.end.toFixed(3)} 秒的完整切片?全部动作描述、无效标记及时间保持不变。\n确定:合并外层;取消:保留原划分。`, + ) + ) + consolidateInvalidSliceIds.push(candidate.id); + } + if ( + candidates.length && + !window.confirm( + consolidateInvalidSliceIds.length + ? `已选择 ${consolidateInvalidSliceIds.length} 处无损合并。提交后重新生成审核视频,原版本保留。确认提交质检?` + : "保留原切片划分,确认提交质检?", + ) + ) + return; const currentTaskId = detail.task.id; const token = beginAnnotationMutation(currentTaskId); if (!token) return; clearEditHistory(); try { const submission = await annotationOperationsApi.submitCombined(detail.task.id, { + consolidateInvalidSliceIds, version: detailRef.current?.humanItem?.version ?? detail.humanItem.version, sliceSegments: applySliceStatuses( sliceSegments.map((slice) => ({ ...slice, taskLabelId: "", taskLabel: "" })), diff --git a/src/services/api/annotation-operations.ts b/src/services/api/annotation-operations.ts index 2d41d767..8f6f20a3 100644 --- a/src/services/api/annotation-operations.ts +++ b/src/services/api/annotation-operations.ts @@ -178,6 +178,7 @@ export const annotationOperationsApi = { taskId: string, payload: { version: number; + consolidateInvalidSliceIds?: string[]; sliceSegments: SliceSegment[]; sliceOperations?: SliceOperation[]; segments: AnnotationSegment[]; diff --git a/tests/frontend/invalid-slice-consolidation.test.mjs b/tests/frontend/invalid-slice-consolidation.test.mjs new file mode 100644 index 00000000..ed1a6db0 --- /dev/null +++ b/tests/frontend/invalid-slice-consolidation.test.mjs @@ -0,0 +1,40 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { invalidSliceConsolidationCandidates as candidates } from "../../src/lib/invalid-slice-consolidation.js"; +const slices = ["a", "b", "c"].map((id, i) => ({ id, name: "盘点", start: i, end: i + 1 })); +const actions = [{ id: "invalid", start: 1, end: 2, isInvalid: true, description: "无效片段" }]; +test("identify invalid-only outer partition without modifying slices or action JSON", () => { + const before = JSON.stringify({ slices, actions }); + assert.deepEqual(candidates(slices, actions), [ + { id: "b", name: "盘点", start: 0, end: 3, invalidStart: 1, invalidEnd: 2 }, + ]); + assert.equal(JSON.stringify({ slices, actions }), before); +}); +test("never propose discarded, different-name, gapped or partially invalid partitions", () => { + assert.deepEqual( + candidates( + slices.map((s) => (s.id === "b" ? { ...s, deliveryDisposition: "discard" } : s)), + actions, + ), + [], + ); + assert.deepEqual( + candidates( + slices.map((s) => (s.id === "c" ? { ...s, name: "清洁" } : s)), + actions, + ), + [], + ); + assert.deepEqual( + candidates( + slices.map((s) => (s.id === "c" ? { ...s, start: 2.1 } : s)), + actions, + ), + [], + ); + assert.deepEqual(candidates(slices, [{ ...actions[0], end: 1.5 }]), []); + assert.deepEqual(candidates(slices, [{ ...actions[0], isInvalid: false }]), []); +}); +test("invalid inside a single business slice stays inside it", () => { + assert.deepEqual(candidates([{ id: "a", name: "盘点", start: 0, end: 3 }], actions), []); +}); From 74a7d7b00b6f17e27b9cc5813e5957ff547ab7d0 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 24 Sep 2026 18:20:54 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=90=9B=20Return=20rejected=20annotati?= =?UTF-8?q?on=20tasks=20to=20eligible=20original=20authors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/api/annotation_operations.go | 15 ++++- internal/api/review_decision_plan.go | 13 ++++ internal/api/rework_assignment.go | 41 +++++++++++++ internal/api/rework_assignment_test.go | 85 ++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 internal/api/rework_assignment.go create mode 100644 internal/api/rework_assignment_test.go diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index f9cd9915..90ebf469 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -3023,6 +3023,15 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri server.planQualityReviewDecision(writer, request, *task, *item, batch, payload) return true } + returnAuthor := "" + if payload.Decision == domain.AnnotationReviewRejected { + var err error + returnAuthor, err = server.reworkAuthor(request.Context(), *item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return true + } + } now := nowISO() review := domain.AnnotationReviewRecord{ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: task.CurrentRevision, Stage: domain.AnnotationReviewStageQuality, Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, ProblemSliceIDs: payload.ProblemSliceIDs, CreatedAt: now} review.ReviewRound = item.ReviewRound @@ -3030,7 +3039,7 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri review.NextRevision = task.CurrentRevision + 1 review.RejectTargets = []string{"annotation_combined"} task.Status = domain.AnnotationTaskStatusRework - task.AssignedUserID = "" + task.AssignedUserID = returnAuthor } else { // Quality approval is not packaging approval. The root task is accepted // and waits for the one aggregate pre-package acceptance item. @@ -3062,6 +3071,10 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri _ = server.store.UpsertAnnotationTask(request.Context(), *task) } else { item.Status = domain.HumanItemRework + item.AssignedUserID = returnAuthor + if returnAuthor != "" { + item.ClaimedAt = now + } } } else { item.Status = domain.HumanItemAccepted diff --git a/internal/api/review_decision_plan.go b/internal/api/review_decision_plan.go index 60548ade..9b55c97a 100644 --- a/internal/api/review_decision_plan.go +++ b/internal/api/review_decision_plan.go @@ -8,6 +8,15 @@ import ( ) func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Request, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, batch *domain.HumanBatchRecord, payload annotationReviewInput) { + returnAuthor := "" + if payload.Decision == domain.AnnotationReviewRejected { + var err error + returnAuthor, err = s.reworkAuthor(r.Context(), item) + if err != nil { + renderError(w, http.StatusInternalServerError, err) + return + } + } op := operationFromRequest(r) now := op.CreatedAt beforeTask, beforeItem := task, item @@ -26,6 +35,10 @@ func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Reques item.ReviewRound++ item.CurrentStageID = "annotation_combined" item.Status = domain.HumanItemRework + item.AssignedUserID, task.AssignedUserID = returnAuthor, returnAuthor + if returnAuthor != "" { + item.ClaimedAt = now + } if item.ReviewRound > item.MaxReworkRounds { item.Status = domain.HumanItemEscalated task.Status = domain.AnnotationTaskStatusRejected diff --git a/internal/api/rework_assignment.go b/internal/api/rework_assignment.go new file mode 100644 index 00000000..b4aa7ba4 --- /dev/null +++ b/internal/api/rework_assignment.go @@ -0,0 +1,41 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "strings" +) + +// Resolve before recording a decision: transient lookup failures must not silently +// turn a return-to-author decision into a public-pool assignment. +func (s *server) reworkAuthor(ctx context.Context, item domain.HumanItemRecord) (string, error) { + if item.LatestAuthorUserID == "" || item.ReviewRound+1 > item.MaxReworkRounds { + return "", nil + } + user, err := s.store.GetUser(ctx, item.LatestAuthorUserID) + if err != nil { + return "", err + } + if user == nil || user.OrganizationID != item.OrganizationID || user.Status != domain.UserStatusActive || user.Role != domain.UserRoleAnnotator || user.SupplierID == "" { + return "", nil + } + if item.SupplierID != "" && item.SupplierID != user.SupplierID { + return "", nil + } + supplier, err := s.store.GetSupplier(ctx, user.SupplierID) + if err != nil { + return "", err + } + if supplier == nil || supplier.OrganizationID != item.OrganizationID || !strings.EqualFold(supplier.Status, "active") { + return "", nil + } + members, err := s.annotationMembershipsForUser(ctx, item.OrganizationID, item.ProjectID, user.ID) + if err != nil { + return "", err + } + task := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, ProcessingScope: item.ProcessingScope} + if !membershipHasRole(members, task, domain.AnnotationRoleAnnotator) { + return "", nil + } + return user.ID, nil +} diff --git a/internal/api/rework_assignment_test.go b/internal/api/rework_assignment_test.go new file mode 100644 index 00000000..06b20c51 --- /dev/null +++ b/internal/api/rework_assignment_test.go @@ -0,0 +1,85 @@ +package api + +import ( + "droboticflow/internal/domain" + "encoding/json" + "net/http" + "testing" +) + +func TestQualityRejectionReturnsToEligibleAuthor(t *testing.T) { + for _, durable := range []bool{false, true} { + for _, scenario := range []string{"eligible", "disabled", "wrong-supplier", "no-membership", "supplier-inactive", "no-author", "escalated"} { + t.Run(scenario+map[bool]string{true: "-durable", false: "-legacy"}[durable], func(t *testing.T) { + f := decisionFixture(t) + f.item.LatestAuthorUserID = f.annotator.ID + f.item.SupplierID = f.annotator.SupplierID + f.task.SupplierID = f.annotator.SupplierID + f.item.MaxReworkRounds = 5 + switch scenario { + case "disabled": + f.annotator.Status = domain.UserStatusDisabled + f.store.UpsertUser(f.ctx, f.annotator) + case "wrong-supplier": + f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: "other", OrganizationID: f.item.OrganizationID, Name: "Other", Status: "active"}) + f.annotator.SupplierID = "other" + f.annotator.ID = "different-supplier-author" + f.item.LatestAuthorUserID = f.annotator.ID + f.store.UpsertUser(f.ctx, f.annotator) + case "no-membership": + f.store.DeleteProjectMember(f.ctx, "member-"+f.annotator.ID) + case "supplier-inactive": + f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: f.annotator.SupplierID, OrganizationID: f.item.OrganizationID, Name: "Inactive", Status: "inactive"}) + case "no-author": + f.item.LatestAuthorUserID = "" + case "escalated": + f.item.MaxReworkRounds = 1 + } + f.store.UpsertHumanItem(f.ctx, f.item) + f.store.UpsertAnnotationTask(f.ctx, f.task) + input := map[string]any{"stage": "quality", "decision": "rejected", "reasonCodes": []string{"manual_review_reject"}, "comment": "请修正", "problemSliceIds": []string{"left"}} + endpoint := "/api/plugins/annotation-qa/tasks/" + f.task.ID + "/reviews" + var code int + var body string + if durable { + version, headers := openReviewDraftSession(t, f) + input["operationId"] = "return-author" + input["draftVersion"] = 0 + input["resultVersion"] = version + r := reviewSessionRequest(f, http.MethodPost, endpoint, f.reviewer, input, headers) + code = r.Code + body = r.Body.String() + } else { + r := f.request(http.MethodPost, endpoint, f.reviewer, input) + code = r.Code + body = r.Body.String() + } + if code != 201 { + t.Fatalf("review %d %s", code, body) + } + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + task, _ := f.store.GetAnnotationTask(f.ctx, f.task.ID) + expected := "" + if scenario == "eligible" { + expected = f.annotator.ID + } + if item.AssignedUserID != expected || task.AssignedUserID != expected { + t.Fatalf("return owner item=%q task=%q want=%q", item.AssignedUserID, task.AssignedUserID, expected) + } + if item.CurrentRevision != f.item.CurrentRevision { + t.Fatal("annotation revision changed") + } + if expected != "" { + r := f.request(http.MethodGet, "/api/human-items?pool=mine&projectId="+item.ProjectID, f.annotator, nil) + var p struct { + Items []domain.HumanItemRecord `json:"items"` + } + json.Unmarshal(r.Body.Bytes(), &p) + if len(p.Items) != 1 { + t.Fatalf("returned task missing in author's pool: %s", r.Body.String()) + } + } + }) + } + } +} From 15fc98ef3e018c707594188dab6b00fc9006d51d Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sat, 26 Sep 2026 05:23:17 -0700 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=90=9B=20Stream=20authorized=20video?= =?UTF-8?q?=20directly=20from=20BOS=20and=20recover=20playback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../video-presigned-playback-20260926.md | 59 ++++++ internal/api/object_storage_browser.go | 10 ++ internal/api/server_test.go | 2 +- internal/api/video_playback_redirect_test.go | 86 +++++++++ src/components/CaptureAnnotationWorkbench.tsx | 23 ++- src/hooks/useCaptureVideoTransport.ts | 81 ++++++++- tests/browser/video-playback-recovery.mjs | 170 ++++++++++++++++++ .../capture-annotation-workbench.test.mjs | 4 +- 8 files changed, 421 insertions(+), 14 deletions(-) create mode 100644 docs/verification/video-presigned-playback-20260926.md create mode 100644 internal/api/video_playback_redirect_test.go create mode 100644 tests/browser/video-playback-recovery.mjs diff --git a/docs/verification/video-presigned-playback-20260926.md b/docs/verification/video-presigned-playback-20260926.md new file mode 100644 index 00000000..021c3e3d --- /dev/null +++ b/docs/verification/video-presigned-playback-20260926.md @@ -0,0 +1,59 @@ +# Video playback: signed BOS delivery and recovery + +## Problem and final behavior + +Video Range requests were copied through the API's shared public egress and +two-minute response deadline. Focused single-camera inspection hid the other +video visually but continued loading and playing it. After a native media +failure, seeking cleared the message without reloading the failed element. + +- Authorized BOS video GETs now return a non-cacheable 307 to the existing + one-hour, read-only presigned URL, including valid Range requests. The browser + reads video bytes from BOS. Task/object authorization remains unchanged. +- HEAD, invalid ranges, explicit transcoding, structured previews, local files, + and backends without presigning retain their existing behavior. +- Focused single-camera inspection attaches only the selected source. Camera + changes align to the existing logical frame clock; clicking the already + selected camera does not interrupt playback. +- Media failures get at most two automatic reload attempts per unsuccessful + recovery. Seeking or the inline reload control also retries. Reloading uses the + original authorized route with a cache-busting parameter to obtain a fresh + signature, rather than reusing the redirected URL. External signatures are not + modified. Errors clear only once the requested frame is ready. +- Playback intent, the logical position and the mounted editor survive reloads. + No task records, slice boundaries, action judgments, source media or delivery + JSON are changed by this fix. + +## Verification + +The backend regression initially observed HTTP 206 instead of 307. The real +component browser regression initially found a src on the hidden right camera. +Both pass after the fix. + +- Backend object storage content, signed Range routing, anonymous authorization, + task evidence isolation, run artifact content and preview compatibility tests + pass. +- TypeScript check and production web build pass. +- Frontend suite: 612 tests, 599 passing, the same 13 pre-existing failures before + and after this change; failure names are recorded in the private release log. +- `tests/browser/video-playback-recovery.mjs` uses local synthetic video and the + real shared workbench in annotation, quality and acceptance viewing modes. It + checks hidden-stream absence, camera switching, repeated selection, transient + and persistent failures, bounded retries, frame positioning, explicit retry + and unsaved description preservation. +- Pre-release read-only browser checks confirmed actual BOS signed redirects + play successfully. All customer write requests were blocked before sending. + +## Release and rollback + +The user authorized this release and requested backups. Local private release +artifacts are under `local-data/verification/presigned-video-20260926/` in the +primary repository. Deployment, service and configuration snapshots retain +their pre-release versions. The API is updated by immutable image digest only, +with a compare-and-swap guard against overwriting a concurrent release. Other +deployments and customer data are outside this release. + +The saved inverse image patch is the rollback operation. Production verification +must remain read-only; report observed capacity separately from a full 12-user +business-workflow acceptance. No pull request or Git push is included in this +authorization. diff --git a/internal/api/object_storage_browser.go b/internal/api/object_storage_browser.go index 4aa13082..fe3c77ca 100644 --- a/internal/api/object_storage_browser.go +++ b/internal/api/object_storage_browser.go @@ -2358,6 +2358,16 @@ func (server *server) handleObjectStorageContent(writer http.ResponseWriter, req renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr) return } + // Browser media requests normally include Range, including open-ended + // ranges. Redirect those too: proxying them shares the API's public + // bandwidth and fixed response deadline across all active viewers. + // Authorization and object resolution above still precede URL issuance. + if request.Method == http.MethodGet && summary.Kind == "video" && !previewOnly && redirectURL != "" { + writer.Header().Set("Cache-Control", "private, no-store") + writer.Header().Set("Referrer-Policy", "no-referrer") + http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) + return + } if request.Method == http.MethodHead || hasRange { if !hasRange && summary.SizeBytes > 0 { requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1} diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 3b853094..6fa3fd93 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -10449,7 +10449,7 @@ func TestObjectStorageContentSupportsRangeAndHeadWithoutReadingWholeObject(t *te if item == nil || item.ID != connection.ID { t.Fatalf("unexpected storage connection: %#v", item) } - return fakeBrowser, func() {}, nil + return &unsignedVideoBrowser{fakeBrowser}, func() {}, nil } defer func() { storageConnectionBrowserResolver = originalResolver }() diff --git a/internal/api/video_playback_redirect_test.go b/internal/api/video_playback_redirect_test.go new file mode 100644 index 00000000..c62bde70 --- /dev/null +++ b/internal/api/video_playback_redirect_test.go @@ -0,0 +1,86 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// A backend without presigning still supports the existing Range proxy. +type unsignedVideoBrowser struct{ *fakeObjectStorageBrowser } + +func (b *unsignedVideoBrowser) BuildPreviewURL(context.Context, string, string) (string, error) { + return "", nil +} + +func TestVideoPlaybackRedirectsRangesWithoutProxyingBytes(t *testing.T) { + ctx := context.Background() + data := store.NewMemoryStore() + connection, err := data.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "video-test", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test", + Bucket: "videos", AccessKeyID: "test-ak", SecretAccessKey: "test-sk", + }) + if err != nil { + t.Fatal(err) + } + browser := newFakeObjectStorageBrowser() + key := "videos|capture.mp4" + browser.objects[key] = domain.ObjectStorageObjectSummary{ + Bucket: "videos", Key: "capture.mp4", Name: "capture.mp4", Kind: "video", + SizeBytes: 10, ContentType: "video/mp4", + } + browser.content[key] = []byte("0123456789") + old := storageConnectionBrowserResolver + defer func() { storageConnectionBrowserResolver = old }() + storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + return browser, nil, nil + } + handler := NewHandler(data, nil) + path := "/api/object-storage/content?connectionId=" + url.QueryEscape(connection.ID) + "&bucket=videos&key=capture.mp4" + for _, byteRange := range []string{"bytes=0-", "bytes=2-5", "bytes=-4", ""} { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("Range", byteRange) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusTemporaryRedirect { + t.Fatalf("range %q: status %d, want 307", byteRange, response.Code) + } + if response.Header().Get("Location") != "https://example.com/preview/videos/capture.mp4" { + t.Fatal("expected presigned storage target") + } + if response.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal("signed redirects must not be cached") + } + } + if browser.readCalls[key] != 0 || browser.readRangeCalls[key] != 0 { + t.Fatal("API must not transfer any video bytes") + } + // Invalid ranges remain rejected locally; metadata-only reads remain HEAD. + for _, tc := range []struct { + method, byteRange string + status int + }{ + {http.MethodGet, "bytes=20-", 416}, {http.MethodHead, "", 200}, + } { + request := httptest.NewRequest(tc.method, path, nil) + request.Header.Set("Range", tc.byteRange) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != tc.status || response.Header().Get("Location") != "" { + t.Fatalf("method %s range %q: status %d", tc.method, tc.byteRange, response.Code) + } + } + // The original authorization gate is still in front of URL issuance. + t.Setenv("DROBOTICFLOW_REQUIRE_SESSION_AUTH", "true") + authenticatedHandler := NewHandler(data, nil) + response := httptest.NewRecorder() + authenticatedHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + if response.Code != http.StatusUnauthorized || response.Header().Get("Location") != "" { + t.Fatalf("anonymous request leaked video URL: status %d", response.Code) + } +} diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index 1d49e9ed..84e45e3e 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -379,8 +379,6 @@ export function CaptureAnnotationWorkbench({ const [playbackRate, setPlaybackRate] = useState(1); const [showLeft, setShowLeft] = useState(initialCameraVisibility?.left ?? true); const [showRight, setShowRight] = useState(initialCameraVisibility?.right ?? Boolean(rightVideoSource)); - const effectiveShowLeft = focusedLayout || showLeft; - const effectiveShowRight = focusedLayout ? Boolean(rightVideoSource) : showRight; const cameraStorageKey = cameraPreferenceKey ? `${isQualityReview || providedReviewCaption ? "review" : "annotation"}-camera-default:${cameraPreferenceKey}` : ""; @@ -394,6 +392,7 @@ export function CaptureAnnotationWorkbench({ }; const [defaultCamera, setDefaultCamera] = useState(readCameraDefault); const initialInspection = () => { + if (!rightVideoSource) return "left" as const; if (providedReviewCaption && initialCameraVisibility) return initialCameraVisibility.left && initialCameraVisibility.right ? null @@ -404,7 +403,11 @@ export function CaptureAnnotationWorkbench({ return preference === "both" ? null : (preference as "left" | "right"); }; const [inspectionCamera, setInspectionCamera] = useState<"left" | "right" | null>(initialInspection); + const effectiveShowLeft = focusedLayout ? inspectionCamera !== "right" || !rightVideoSource : showLeft; + const effectiveShowRight = Boolean(rightVideoSource) && (focusedLayout ? inspectionCamera !== "left" : showRight); const inspectCamera = (camera: "left" | "right" | null) => { + if (camera === inspectionCamera) return; + transport.prepareCameraChange(); setInspectionCamera(camera); if (providedReviewCaption) onCameraVisibilityChange?.({ left: camera !== "right", right: camera !== "left" }); }; @@ -2650,7 +2653,7 @@ export function CaptureAnnotationWorkbench({ )} diff --git a/src/hooks/useCaptureVideoTransport.ts b/src/hooks/useCaptureVideoTransport.ts index 6f41c326..573e5c6a 100644 --- a/src/hooks/useCaptureVideoTransport.ts +++ b/src/hooks/useCaptureVideoTransport.ts @@ -34,6 +34,27 @@ export function useCaptureVideoTransport(options: Options) { const [isPositioning, setIsPositioning] = useState(false); const [mediaError, setMediaError] = useState(""); const targets = useRef<{ video: HTMLVideoElement; time: number; revision: number }[]>([]); + const retryTimer = useRef | null>(null); + const retryAttempts = useRef({ left: 0, right: 0 }); + const reloadSequence = useRef(0); + + function cancelRetry() { + if (retryTimer.current !== null) clearTimeout(retryTimer.current); + retryTimer.current = null; + } + function reloadTrack(video: HTMLVideoElement) { + const c = config.current; + const source = video === c.leftRef.current ? c.leftSource : c.rightSource; + if (!source) return; + const url = new URL(source, window.location.href); + // Return to the original authenticated route, never to currentSrc (which + // can be an expired BOS redirect). Do not alter external URL signatures. + if (url.origin === window.location.origin) { + url.searchParams.set("_mediaRetry", `${Date.now()}-${++reloadSequence.current}`); + } + video.src = url.href; + video.load(); + } function visibleTracks() { const c = config.current; @@ -66,6 +87,10 @@ export function useCaptureVideoTransport(options: Options) { } function seekTrack(target: (typeof targets.current)[number]) { const { video, time } = target; + if (video.error) { + reloadTrack(video); + return; + } if (video.readyState < 1) return; // loadedmetadata will finish this request. const next = physicalTarget(video, time); if (Math.abs(video.currentTime - next) > 0.000001) video.currentTime = next; @@ -77,19 +102,24 @@ export function useCaptureVideoTransport(options: Options) { ({ video, time }) => // HTMLMediaElement can quantize to microseconds. This tolerance is // only for readiness, never for the saved/visible annotation time. - video.seeking || video.readyState < 2 || Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002, + video.error || + video.seeking || + video.readyState < 2 || + Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002, ) ) return; const revision = targets.current[0].revision; targets.current = []; + setMediaError(""); + retryAttempts.current = { left: 0, right: 0 }; queue.complete(revision); } const [queue] = useState(() => createMediaSeekQueue({ dispatch(time: number, revision: number) { + cancelRetry(); setIsPositioning(true); - setMediaError(""); pauseElements(); targets.current = visibleTracks().map(({ video, toMedia }) => ({ video, time: toMedia(time), revision })); targets.current.forEach(seekTrack); @@ -105,6 +135,7 @@ export function useCaptureVideoTransport(options: Options) { ); useEffect( () => () => { + cancelRetry(); playing.current = false; queue.cancel(); targets.current = []; @@ -165,11 +196,25 @@ export function useCaptureVideoTransport(options: Options) { function play() { playing.current = true; setIsPlaying(true); + if (visibleTracks().some(({ video }) => video.error)) { + retry(); + return; + } if (!queue.isPending() && !scrubbing.current) playElements(); } + function retry() { + cancelRetry(); + retryAttempts.current = { left: 0, right: 0 }; + queue.cancel(); + queue.commit(position.current); + } + // Sources/visibility, not playhead updates, own resource lifetime. useEffect(() => { + cancelRetry(); + retryAttempts.current = { left: 0, right: 0 }; + setMediaError(""); queue.cancel(); targets.current = []; scrubbing.current = false; @@ -191,6 +236,7 @@ export function useCaptureVideoTransport(options: Options) { } queue.commit(position.current); return () => { + cancelRetry(); queue.cancel(); targets.current = []; }; @@ -210,12 +256,28 @@ export function useCaptureVideoTransport(options: Options) { onLoadedData: settle, onCanPlay: settle, onError(event: { currentTarget: HTMLVideoElement }) { - if (!event.currentTarget.getAttribute("src")) return; + if ( + !event.currentTarget.getAttribute("src") || + !visibleTracks().some(({ video }) => video === event.currentTarget) + ) + return; + cancelRetry(); queue.cancel(); targets.current = []; setIsPositioning(false); - setMediaError("视频加载失败,请重新定位或重新打开任务。"); - pause(); + setMediaError("视频加载失败,当前编辑已保留。"); + // Keep play intent and the logical position, so recovery does not + // remount the editor or jump to the beginning of the video. + pauseElements(); + if (config.current.enabled && retryAttempts.current[camera] < 2) { + const attempt = ++retryAttempts.current[camera]; + retryTimer.current = setTimeout(() => { + retryTimer.current = null; + queue.commit(position.current); + }, attempt * 750); + } else { + pause(); + } }, onPlay(event: { currentTarget: HTMLVideoElement }) { if (event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current) @@ -224,7 +286,13 @@ export function useCaptureVideoTransport(options: Options) { setIsPlaying(true); }, onPause(event: { currentTarget: HTMLVideoElement }) { - if (!event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current) + if ( + !event.currentTarget.paused || + !isMaster(event.currentTarget) || + queue.isPending() || + scrubbing.current || + retryTimer.current !== null + ) return; pause(); }, @@ -257,6 +325,7 @@ export function useCaptureVideoTransport(options: Options) { pause, pauseAtFrame, play, + retry, isPlaying, isPositioning, mediaError, diff --git a/tests/browser/video-playback-recovery.mjs b/tests/browser/video-playback-recovery.mjs new file mode 100644 index 00000000..ee305420 --- /dev/null +++ b/tests/browser/video-playback-recovery.mjs @@ -0,0 +1,170 @@ +// Local real-component/media regression. Synthetic video only; no API writes. +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { chromium } from "playwright"; +const base = process.env.VIDEO_TEST_URL || "http://127.0.0.1:5196"; +assert(["127.0.0.1", "localhost"].includes(new URL(base).hostname), "local fixtures only"); +const out = process.env.VIDEO_TEST_OUTPUT || "/tmp/presigned-video-tests"; +await fs.mkdir(out, { recursive: true }); +const file = out + "/sample.mp4"; +try { + await fs.access(file); +} catch { + execFileSync("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=240x320:rate=30:duration=12", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + file, + ]); +} +const bytes = await fs.readFile(file); +const browser = await chromium.launch({ + executablePath: + process.env.PLAYWRIGHT_EXECUTABLE_PATH || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, + args: ["--no-proxy-server"], +}); +try { + for (const mode of ["annotation", "quality", "acceptance"]) { + const page = await browser.newPage({ viewport: { width: 1413, height: 900 } }); + page.setDefaultTimeout(15000); + let fail = false, + failedRequests = 0; + const requests = [], + errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + await page.route("**/api/**", async (route) => { + const req = route.request(), + u = new URL(req.url()); + assert.equal(req.method(), "GET"); + if (!u.pathname.endsWith(".mp4")) return route.fulfill({ json: {} }); + requests.push({ path: u.pathname, retry: u.searchParams.get("_mediaRetry") }); + if (fail && u.pathname.includes("left")) { + failedRequests++; + return route.abort("failed"); + } + const match = /bytes=(\d+)-(\d*)/.exec(req.headers().range || ""); + const start = match ? Number(match[1]) : 0, + end = match?.[2] ? Math.min(Number(match[2]), bytes.length - 1) : bytes.length - 1; + return route.fulfill({ + status: match ? 206 : 200, + contentType: "video/mp4", + headers: { + "Accept-Ranges": "bytes", + "Content-Length": String(end - start + 1), + ...(match ? { "Content-Range": `bytes ${start}-${end}/${bytes.length}` } : {}), + }, + body: bytes.subarray(start, end + 1), + }); + }); + await page.route("**/__video_recovery", (route) => + route.fulfill({ + contentType: "text/html", + body: `
`, + }), + ); + await page.goto(base + "/__video_recovery", { waitUntil: "domcontentloaded" }); + const bench = page.getByTestId("capture-annotation-workbench"), + left = page.locator('video[data-camera="left"]'), + right = page.locator('video[data-camera="right"]'); + const settle = () => + page.waitForFunction(() => { + const videos = [...document.querySelectorAll("video")].filter((v) => v.getAttribute("src")); + return videos.length && videos.every((v) => v.readyState >= 2 && !v.seeking && !v.error); + }); + await bench.waitFor(); + await settle(); + assert.equal(await right.getAttribute("src"), null, "single-camera view must detach the hidden stream"); + assert.equal( + requests.filter((r) => r.path.includes("right")).length, + 0, + "hidden camera must make no media request", + ); + await bench.getByRole("button", { name: "播放", exact: true }).click(); + await page.waitForFunction(() => document.querySelector('video[data-camera="left"]').currentTime > 1); + const beforeSameCamera = await left.evaluate((v) => v.currentTime); + await bench.getByRole("button", { name: "放大左相机", exact: true }).click(); + await page.waitForFunction( + (t) => document.querySelector('video[data-camera="left"]').currentTime > t + 0.3, + beforeSameCamera, + ); + // One transient read failure recovers automatically and retains play intent. + fail = true; + await left.evaluate((v) => v.load()); + await page.waitForFunction(() => !!document.querySelector('video[data-camera="left"]').error); + fail = false; + await settle(); + await page.waitForFunction(() => !document.querySelector('video[data-camera="left"]').paused); + failedRequests = 0; + await bench.getByRole("button", { name: "放大右相机", exact: true }).click(); + await settle(); + assert.equal(await left.getAttribute("src"), null); + assert((await right.evaluate((v) => v.currentTime)) > 0.8, "camera switch preserves logical playback position"); + await bench.getByRole("button", { name: "恢复双画面", exact: true }).click(); + await settle(); + await bench.getByRole("button", { name: "暂停", exact: true }).click(); + await bench.getByRole("button", { name: "放大左相机", exact: true }).click(); + await settle(); + const held = await left.evaluate((v) => v.currentTime); + let input; + if (mode === "annotation") { + await bench.getByTestId("annotation-description-block").first().click(); + input = bench.getByRole("textbox", { name: "动作描述", exact: true }); + await input.fill("Unsubmitted draft must survive video recovery"); + } + const beforeFailure = await left.evaluate((v) => v.currentTime); + fail = true; + await left.evaluate((v) => v.load()); + await page.waitForTimeout(4500); + assert(failedRequests <= 3, `automatic retries must be bounded: ${failedRequests}`); + assert(await left.evaluate((v) => !!v.error), "persistent failure remains visible in the media element"); + assert(await bench.getByRole("alert").count(), "do not clear the failure before media recovers"); + fail = false; + await bench.getByRole("button", { name: "下一帧", exact: true }).click(); + await settle(); + assert( + requests.some((r) => r.retry), + "retry returns to original authorized URL for a fresh signature", + ); + assert( + Math.abs((await left.evaluate((v) => v.currentTime)) - beforeFailure) < 0.15, + "recovery preserves the frame clock", + ); + assert.equal(await left.evaluate((v) => v.paused), true, "paused users are not auto-played"); + if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery"); + // Explicit retry also works after the bounded automatic attempts are exhausted. + fail = true; + failedRequests = 0; + await left.evaluate((v) => v.load()); + await page.waitForTimeout(4500); + fail = false; + await bench.getByRole("button", { name: "重新加载视频", exact: true }).click(); + await settle(); + assert.equal(await bench.getByRole("alert").count(), 0); + if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery"); + assert.deepEqual(errors, []); + console.log(JSON.stringify({ mode, passed: true, held, requests: requests.length })); + await page.close(); + } +} finally { + await browser.close(); +} diff --git a/tests/frontend/capture-annotation-workbench.test.mjs b/tests/frontend/capture-annotation-workbench.test.mjs index 8cb0fdd5..ae5d0acd 100644 --- a/tests/frontend/capture-annotation-workbench.test.mjs +++ b/tests/frontend/capture-annotation-workbench.test.mjs @@ -67,8 +67,8 @@ test("both playhead sliders use latest-only pointer previews and exact final com }); test("hidden sources detach and right-only playback maps back to the canonical frame clock", () => { - assert.match(workbench, /const effectiveShowLeft = focusedLayout \|\| showLeft/); - assert.match(workbench, /const effectiveShowRight = focusedLayout \? Boolean\(rightVideoSource\) : showRight/); + // Real focused-camera resource lifetime is exercised by + // tests/browser/video-playback-recovery.mjs (hidden camera issues no request). assert.match(workbench, /src=\{effectiveShowLeft \? source : undefined\}/); assert.match(workbench, /src=\{effectiveShowRight \? rightSource : undefined\}/); assert.match(workbench, /mapCameraBoundaryTime\(rightMediaBoundaries, frameBoundaries, time\)/);