🔀 Sync production video playback fixes into local workspace

# Conflicts:
#	scripts/deploy-envs/fanyu.env
This commit is contained in:
d-robotics 2026-09-26 05:28:14 -07:00
commit c5cdd6c1da
36 changed files with 1740 additions and 125 deletions

View File

@ -0,0 +1,13 @@
# 无效动作与外层切片的无损整理
无效动作属于外层业务切片内部,设置无效不新增外层边界。旧版取消切分仍按既有规则清空合并区间判断;它不是无损整理入口。
送检前识别以下候选:连续三个保留切片名称完全相同、首尾精确相接,中间切片由明确无效的动作完整覆盖。用户逐项选择是否合并,最后再次确认提交。取消选择可保留原划分,不对所有同名切片做自动合并。舍弃、名称不同、缺口、部分无效都不匹配。
服务器重新验证选择,仅在已有领取权、版本、字典、覆盖与动作合法性校验全部通过后,改变送检版本的外层划分。动作数组不因合并改变:ID、描述、无效判断、起止边界、硬件时间戳均保留。重叠候选合成同一连续区间。操作历史记录原切片快照。
后台生成新的媒体和 JSON;不能把旧短片直接冒充合并后视频。JSON 中相对于切片起点的时间随新的切片起点正常重基,源视频时间、硬件时间戳、动作含义保持一致。旧版本文件和记录通过任务级历史保留标记保护,后续送检和重切也不会由普通版本清理删除。
部署无数据库迁移,不自动修改已送检、已审核任务,也不代替客户送检。现有在审任务需要正常退回至标注后使用此入口;无需重新填写动作。更新现有在审结果并保持原审核状态,需要独立的、明确授权的数据修订流程,不属于本次部署。
验证覆盖:候选识别、非法选择拒绝且不占用任务、重叠候选、真实持久队列与媒体生成、导出 JSON 动作与时间戳保留,以及原版本所有文件逐字节未变。

View File

@ -0,0 +1,59 @@
# Video playback: signed BOS delivery and recovery
## Problem and final behavior
Video Range requests were copied through the API's shared public egress and
two-minute response deadline. Focused single-camera inspection hid the other
video visually but continued loading and playing it. After a native media
failure, seeking cleared the message without reloading the failed element.
- Authorized BOS video GETs now return a non-cacheable 307 to the existing
one-hour, read-only presigned URL, including valid Range requests. The browser
reads video bytes from BOS. Task/object authorization remains unchanged.
- HEAD, invalid ranges, explicit transcoding, structured previews, local files,
and backends without presigning retain their existing behavior.
- Focused single-camera inspection attaches only the selected source. Camera
changes align to the existing logical frame clock; clicking the already
selected camera does not interrupt playback.
- Media failures get at most two automatic reload attempts per unsuccessful
recovery. Seeking or the inline reload control also retries. Reloading uses the
original authorized route with a cache-busting parameter to obtain a fresh
signature, rather than reusing the redirected URL. External signatures are not
modified. Errors clear only once the requested frame is ready.
- Playback intent, the logical position and the mounted editor survive reloads.
No task records, slice boundaries, action judgments, source media or delivery
JSON are changed by this fix.
## Verification
The backend regression initially observed HTTP 206 instead of 307. The real
component browser regression initially found a src on the hidden right camera.
Both pass after the fix.
- Backend object storage content, signed Range routing, anonymous authorization,
task evidence isolation, run artifact content and preview compatibility tests
pass.
- TypeScript check and production web build pass.
- Frontend suite: 612 tests, 599 passing, the same 13 pre-existing failures before
and after this change; failure names are recorded in the private release log.
- `tests/browser/video-playback-recovery.mjs` uses local synthetic video and the
real shared workbench in annotation, quality and acceptance viewing modes. It
checks hidden-stream absence, camera switching, repeated selection, transient
and persistent failures, bounded retries, frame positioning, explicit retry
and unsaved description preservation.
- Pre-release read-only browser checks confirmed actual BOS signed redirects
play successfully. All customer write requests were blocked before sending.
## Release and rollback
The user authorized this release and requested backups. Local private release
artifacts are under `local-data/verification/presigned-video-20260926/` in the
primary repository. Deployment, service and configuration snapshots retain
their pre-release versions. The API is updated by immutable image digest only,
with a compare-and-swap guard against overwriting a concurrent release. Other
deployments and customer data are outside this release.
The saved inverse image patch is the rollback operation. Production verification
must remain read-only; report observed capacity separately from a full 12-user
business-workflow acceptance. No pull request or Git push is included in this
authorization.

View File

@ -47,7 +47,7 @@ func TestContinuousBlankActionsReachReviewAndPublishedJSON(t *testing.T) {
}
}
func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation) {
func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation, consolidate ...bool) {
t.Helper()
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
@ -103,6 +103,11 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
}
}
slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: 4}}
var selected []string
if len(consolidate) > 0 && consolidate[0] {
slices = []domain.SliceSegment{{ID: "left", Name: "测试切片", Start: 0, End: 1}, {ID: "invalid", Name: "测试切片", Start: 1, End: 2}, {ID: "right", Name: "测试切片", Start: 2, End: 4}}
selected = []string{"invalid"}
}
version := f.item.Version
if reuse {
response := f.request(http.MethodPost, endpoint+"/slice-confirmations", f.annotator, map[string]any{"version": version, "sliceSegments": slices})
@ -111,14 +116,40 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
}
version = waitFor(false)
}
priorFiles := map[string]string{}
if reuse && len(selected) > 0 {
priorRoot := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, "revision-000001")
err := filepath.Walk(priorRoot, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
data, e := os.ReadFile(path)
if e != nil {
return e
}
priorFiles[path] = string(data)
}
return nil
})
if err != nil || len(priorFiles) == 0 {
t.Fatalf("prior evidence unavailable: %v", err)
}
}
response := f.request(http.MethodPost, endpoint+"/combined-submissions", f.annotator, map[string]any{
"version": version, "requireSliceCompletion": true,
"sliceSegments": slices, "segments": segments,
"sliceSegments": slices, "segments": segments, "consolidateInvalidSliceIds": selected,
})
if response.Code != http.StatusAccepted {
t.Fatalf("blank submission: %d %s", response.Code, response.Body.String())
}
waitFor(true)
for path, original := range priorFiles {
data, e := os.ReadFile(path)
if e != nil || string(data) != original {
t.Fatalf("old revision overwritten: %s %v", path, e)
}
}
response = f.request(http.MethodGet, endpoint, f.reviewer, nil)
if response.Code != http.StatusOK {
t.Fatalf("quality read: %d %s", response.Code, response.Body.String())
@ -149,7 +180,7 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
if reuse {
expectedGeneration = 2
}
if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != reuse {
if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != (reuse && len(selected) == 0) {
t.Fatalf("expected generation %d reuse %v: %s", expectedGeneration, reuse, response.Body.String())
}
path := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, fmt.Sprintf("revision-%06d", generation), "slices", "slice-000001", "annotation.json")
@ -180,3 +211,9 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
}
}
}
func TestInvalidSliceConsolidationRebuildsMediaAndPreservesPublishedActions(t *testing.T) {
actions := []domain.AnnotationSegment{{ID: "left-action", Start: 0, End: 1, Status: "described", Description: "拿起"}, {ID: "invalid-action", Start: 1, End: 2, Status: "invalid", IsInvalid: true, Description: "无效片段"}, {ID: "right-action", Start: 2, End: 4, Status: "described", Description: "放下"}}
want := []blankActionExpectation{{"left-action", "described", "拿起", false, 0, 1, 1000000, 2000000}, {"invalid-action", "invalid", "无效片段", true, 1, 2, 2000000, 3000000}, {"right-action", "described", "放下", false, 2, 4, 3000000, 5000000}}
checkBlankActionsDelivery(t, true, actions, want, true)
}

View File

@ -23,6 +23,11 @@ func (s *server) retainCurrentAnnotationData(ctx context.Context, task domain.An
if item.LegacyTaskID != task.ID || item.OrganizationID != task.OrganizationID || item.RunID != task.RunID {
return fmt.Errorf("annotation retention scope mismatch")
}
// A lossless consolidation pins its original evidence for future reslicing
// and submissions, not just the first post-consolidation cleanup.
if item.Subject.Payload["preserveAnnotationRevisionHistory"] == true {
return nil
}
// Disposition history is review evidence. Keep its private prior generations;
// this does not add selectable delivery versions or alter project deletion.
if history, ok := item.Subject.Payload["sliceDispositionHistory"]; ok && history != nil {

View File

@ -0,0 +1,82 @@
package api
import (
"fmt"
"sort"
"strings"
"droboticflow/internal/domain"
)
// Consolidation is an explicit submission choice, never a migration or a
// cancel-cut. Action records are not passed by pointer or rewritten here.
func consolidateInvalidActionSlices(input []domain.SliceSegment, actions []domain.AnnotationSegment, selected []string) ([]domain.SliceSegment, []domain.SliceOperation, error) {
if len(selected) == 0 {
return input, nil, nil
}
ordered := append([]domain.SliceSegment(nil), input...)
sort.Slice(ordered, func(i, j int) bool { return ordered[i].Start < ordered[j].Start })
eligible := map[string]int{}
for i := 1; i < len(ordered)-1; i++ {
l, c, r := ordered[i-1], ordered[i], ordered[i+1]
if l.IsDiscarded() || c.IsDiscarded() || r.IsDiscarded() || strings.TrimSpace(c.Name) == "" || l.Name != c.Name || r.Name != c.Name || l.End != c.Start || c.End != r.Start {
continue
}
cursor, has, valid := c.Start, false, true
inner := append([]domain.AnnotationSegment(nil), actions...)
sort.Slice(inner, func(i, j int) bool { return inner[i].Start < inner[j].Start })
for _, a := range inner {
if a.End <= c.Start || a.Start >= c.End {
continue
}
if a.Start != cursor || a.End > c.End || a.End <= a.Start || !a.IsInvalid {
valid = false
break
}
cursor, has = a.End, true
}
if valid && has && cursor == c.End {
eligible[c.ID] = i
}
}
edges := make([]bool, len(ordered))
seen := map[string]bool{}
for _, id := range selected {
i, ok := eligible[id]
if !ok || seen[id] {
return nil, nil, fmt.Errorf("所选无效区间的相邻切片已变化,请重新核对后提交")
}
seen[id] = true
edges[i-1], edges[i] = true, true
}
var result []domain.SliceSegment
var operations []domain.SliceOperation
for i := 0; i < len(ordered); {
j := i
for j < len(ordered)-1 && edges[j] {
j++
}
merged := ordered[i]
if j > i {
// Keep original IDs in audit provenance; reuse the leftmost stable ID.
merged.SourceIDs = nil
provenance := map[string]bool{}
ids := []string{}
for k := i; k <= j; k++ {
ids = append(ids, ordered[k].ID)
for _, id := range append([]string{ordered[k].ID}, ordered[k].SourceIDs...) {
if !provenance[id] {
merged.SourceIDs = append(merged.SourceIDs, id)
provenance[id] = true
}
}
}
merged.End = ordered[j].End
merged.EndTimestampUS = ordered[j].EndTimestampUS
operations = append(operations, domain.SliceOperation{Type: "consolidate_invalid_actions", SegmentIDs: ids, Before: append([]domain.SliceSegment(nil), ordered[i:j+1]...), After: []domain.SliceSegment{merged}})
}
result = append(result, merged)
i = j + 1
}
return result, operations, nil
}

View File

@ -0,0 +1,82 @@
package api
import (
"encoding/json"
"reflect"
"testing"
"droboticflow/internal/domain"
)
func TestInvalidSliceConsolidationPreservesActionsAndAudit(t *testing.T) {
slices := []domain.SliceSegment{{ID: "a", Name: "收银", Start: 0, End: 2, StartTimestampUS: 100}, {ID: "b", Name: "收银", Start: 2, End: 3}, {ID: "c", Name: "收银", Start: 3, End: 5, EndTimestampUS: 900}, {ID: "d", Name: "其他", Start: 5, End: 9}}
actions := []domain.AnnotationSegment{{ID: "x", Start: 0, End: 2, Description: "拿起商品", Phrases: []string{"细节"}}, {ID: "y", Start: 2, End: 3, IsInvalid: true, Description: "无效片段", Status: "invalid", StartTimestampUS: 200, EndTimestampUS: 300}, {ID: "z", Start: 3, End: 5, Description: "放下商品"}}
before, _ := json.Marshal(actions)
original, _ := json.Marshal(slices)
got, ops, err := consolidateInvalidActionSlices(slices, actions, []string{"b"})
if err != nil || len(got) != 2 || got[0].Start != 0 || got[0].End != 5 || got[0].Name != "收银" || got[0].StartTimestampUS != 100 || got[0].EndTimestampUS != 900 {
t.Fatalf("%+v %v", got, err)
}
after, _ := json.Marshal(actions)
if string(before) != string(after) {
t.Fatal("action JSON changed")
}
unchanged, _ := json.Marshal(slices)
if string(original) != string(unchanged) {
t.Fatal("input slices mutated")
}
if !reflect.DeepEqual(ops[0].Before, slices[:3]) || ops[0].Type != "consolidate_invalid_actions" {
t.Fatal("audit lost")
}
if !reflect.DeepEqual(got[1], slices[3]) {
t.Fatal("unrelated slice changed")
}
kept, ops, err := consolidateInvalidActionSlices(slices, actions, nil)
if err != nil || len(ops) != 0 || !reflect.DeepEqual(kept, slices) {
t.Fatal("implicit consolidation")
}
for name, change := range map[string]func([]domain.SliceSegment, []domain.AnnotationSegment){
"discard": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[1].DeliveryDisposition = "discard" },
"different-name": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Name = "盘点" },
"gap": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Start = 3.1 },
"valid-action": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].IsInvalid = false },
"partial-invalid": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].End = 2.5 },
} {
t.Run(name, func(t *testing.T) {
s := append([]domain.SliceSegment(nil), slices...)
a := append([]domain.AnnotationSegment(nil), actions...)
change(s, a)
if _, _, err := consolidateInvalidActionSlices(s, a, []string{"b"}); err == nil {
t.Fatal("unsafe consolidation accepted")
}
})
}
if _, _, err = consolidateInvalidActionSlices(slices, actions, []string{"b", "b"}); err == nil {
t.Fatal("duplicate accepted")
}
}
func TestInvalidSliceConsolidationOverlappingCandidates(t *testing.T) {
var ss []domain.SliceSegment
var aa []domain.AnnotationSegment
for i, id := range []string{"a", "b", "c", "d", "e"} {
ss = append(ss, domain.SliceSegment{ID: id, Name: "盘点", Start: float64(i), End: float64(i + 1)})
aa = append(aa, domain.AnnotationSegment{ID: id, Start: float64(i), End: float64(i + 1), IsInvalid: i == 1 || i == 3})
}
got, ops, err := consolidateInvalidActionSlices(ss, aa, []string{"b", "d"})
if err != nil || len(got) != 1 || got[0].End != 5 || len(ops) != 1 || len(ops[0].Before) != 5 {
t.Fatalf("%+v %v", got, err)
}
}
func TestInvalidSliceConsolidationRejectsStaleSelectionBeforeReservation(t *testing.T) {
f := continuousFixture(t)
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": []domain.SliceSegment{{ID: "s", Name: "测试切片", Start: 0, End: 10}}, "segments": []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Description: "动作"}}, "consolidateInvalidSliceIds": []string{"missing"}})
if r.Code != 400 {
t.Fatalf("%d %s", r.Code, r.Body.String())
}
item, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
if item.Version != f.item.Version || item.PendingCombinedSubmissionID != "" {
t.Fatal("invalid request mutated task")
}
}

View File

@ -1055,6 +1055,7 @@ type sliceRevisionInput struct {
// Slice correction and semantic labels are persisted together only after both
// validate, so a quality reviewer never receives one without the other.
type combinedAnnotationSubmissionInput struct {
ConsolidateInvalidSliceIDs []string `json:"consolidateInvalidSliceIds,omitempty"`
// Set only by the internal cut job; never accepted from JSON.
sliceOnly bool
Version int `json:"version"`
@ -2056,6 +2057,22 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
renderValidationError(writer, issues...)
return
}
// Original slices, names, disposition and actions have all been validated.
// Only an explicit user selection may change the outer partition. Retain
// action IDs, descriptions, invalid flags and absolute timestamp boundaries.
if len(payload.ConsolidateInvalidSliceIDs) > 0 {
merged, operations, mergeErr := consolidateInvalidActionSlices(payload.SliceSegments, payload.Segments, payload.ConsolidateInvalidSliceIDs)
if mergeErr != nil {
renderValidationError(writer, validationIssue{Field: "consolidateInvalidSliceIds", Message: mergeErr.Error()})
return
}
if issues := validateDescriptionIntervals(merged, payload.Segments, false); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
payload.SliceSegments = merged
payload.SliceOperations = append(payload.SliceOperations, operations...)
}
payloadSHA, hashErr := fanyuV5CombinedPayloadSHA(payload)
if hashErr != nil {
renderError(writer, http.StatusInternalServerError, hashErr)
@ -2201,6 +2218,9 @@ func (server *server) completeCombinedAnnotationSubmission(writer http.ResponseW
if humanItem.Subject.Payload == nil {
humanItem.Subject.Payload = map[string]any{}
}
if len(payload.ConsolidateInvalidSliceIDs) > 0 {
humanItem.Subject.Payload["preserveAnnotationRevisionHistory"] = true
}
humanItem.Subject.Payload["materialization"] = materialization.Payload
if workflow := mapValue(humanItem.Subject.Payload["sliceWorkflow"]); len(workflow) > 0 {
workflow["materialization"] = materialization.Payload
@ -2999,7 +3019,19 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri
return true
}
}
if operationFromRequest(request) != nil { server.planQualityReviewDecision(writer, request, *task, *item, batch, payload); return true }
if operationFromRequest(request) != nil {
server.planQualityReviewDecision(writer, request, *task, *item, batch, payload)
return true
}
returnAuthor := ""
if payload.Decision == domain.AnnotationReviewRejected {
var err error
returnAuthor, err = server.reworkAuthor(request.Context(), *item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return true
}
}
now := nowISO()
review := domain.AnnotationReviewRecord{ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: task.CurrentRevision, Stage: domain.AnnotationReviewStageQuality, Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, ProblemSliceIDs: payload.ProblemSliceIDs, CreatedAt: now}
review.ReviewRound = item.ReviewRound
@ -3007,7 +3039,7 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri
review.NextRevision = task.CurrentRevision + 1
review.RejectTargets = []string{"annotation_combined"}
task.Status = domain.AnnotationTaskStatusRework
task.AssignedUserID = ""
task.AssignedUserID = returnAuthor
} else {
// Quality approval is not packaging approval. The root task is accepted
// and waits for the one aggregate pre-package acceptance item.
@ -3039,6 +3071,10 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri
_ = server.store.UpsertAnnotationTask(request.Context(), *task)
} else {
item.Status = domain.HumanItemRework
item.AssignedUserID = returnAuthor
if returnAuthor != "" {
item.ClaimedAt = now
}
}
} else {
item.Status = domain.HumanItemAccepted
@ -3131,14 +3167,18 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r
renderJSON(writer, http.StatusForbidden, map[string]string{"error": string(requiredRole) + " assignment is required"})
return
}
if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) { return }
if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) {
return
}
humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "")
if !claimed {
return
}
var prepared bool
request, prepared = server.prepareReviewDecision(writer, request, *humanItem, "quality", payload.ReviewDecisionInput, payload, string(payload.Decision))
if !prepared { return }
if !prepared {
return
}
defer server.finishReviewDecisionPreparation(request)
finishReviewWrite, writeAllowed := server.beginReviewWrite(writer, request, *humanItem)
if !writeAllowed {

View File

@ -2358,6 +2358,16 @@ func (server *server) handleObjectStorageContent(writer http.ResponseWriter, req
renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr)
return
}
// Browser media requests normally include Range, including open-ended
// ranges. Redirect those too: proxying them shares the API's public
// bandwidth and fixed response deadline across all active viewers.
// Authorization and object resolution above still precede URL issuance.
if request.Method == http.MethodGet && summary.Kind == "video" && !previewOnly && redirectURL != "" {
writer.Header().Set("Cache-Control", "private, no-store")
writer.Header().Set("Referrer-Policy", "no-referrer")
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
if request.Method == http.MethodHead || hasRange {
if !hasRange && summary.SizeBytes > 0 {
requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1}

View File

@ -18,7 +18,9 @@ import (
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
@ -452,9 +454,14 @@ func (s *server) deleteProjectTarget(ctx context.Context, plan domain.ProjectDel
}
return nil
case "bos":
if err := s.validateDeletionReferences(ctx, plan); err != nil {
return err
}
// validateDeletionReferences is already re-run once by
// executeProjectDeletion immediately before it starts processing
// non-runtime targets (the runtimeDone gate) — that is the right
// point to catch a run created between plan-build and execution
// referencing this path. Re-running the whole org-wide validation
// again here, per bos target, made a project with a few thousand
// BOS targets take hours (each call re-fetches and re-scans every
// other run in the organization).
browser, conn, cleanup, err := s.resolveObjectStorageBrowser(ctx, t.ConnectionID)
if err != nil {
return err
@ -641,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s
}
}
}
runtimeDone := false
for index := range task.Plan.Targets {
target := &task.Plan.Targets[index]
if target.Status == "done" {
continue
targetConcurrency := 12
if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" {
if n, e := strconv.Atoi(v); e == nil && n > 0 {
targetConcurrency = n
}
if target.Kind != "runtime" && !runtimeDone {
deadline := time.Now().Add(150 * time.Second)
for {
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
if e != nil {
fail(e)
return
}
busy := false
for _, id := range task.Plan.RunIDs {
run, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if run != nil && run.ExecutionLeaseOwner != "" {
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
if e != nil || until.After(time.Now()) {
busy = true
}
}
}
for i := range items {
item := &items[i]
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
fail(e)
return
}
w := mapValue(item.Subject.Payload["sliceWorkflow"])
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
busy = true
}
}
if !busy {
break
}
if time.Now().After(deadline) {
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
return
}
select {
case <-ctx.Done():
fail(ctx.Err())
return
case <-time.After(2 * time.Second):
}
}
// processBatch runs deleteProjectTarget for the not-yet-done targets at
// indices with up to targetConcurrency in flight at once. Each target's
// Status/Error and every persist() call is guarded by mu, so concurrent
// workers never race on shared task state; deleteProjectTarget itself
// (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs
// unlocked. A single project's deletion used to process every target
// strictly one at a time, which made stopping thousands of real runs
// take as long as thousands of sequential K8s round trips.
var mu sync.Mutex
processBatch := func(indices []int) error {
sem := make(chan struct{}, targetConcurrency)
var wg sync.WaitGroup
var firstErr error
for _, index := range indices {
mu.Lock()
skip := task.Plan.Targets[index].Status == "done" || firstErr != nil
mu.Unlock()
if skip {
continue
}
// Catch work admitted immediately before the project lock.
for _, step := range task.Plan.Targets {
if step.Kind == "runtime" {
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
fail(e)
return
}
sem <- struct{}{}
wg.Add(1)
go func(index int) {
defer wg.Done()
defer func() { <-sem }()
mu.Lock()
if firstErr != nil {
mu.Unlock()
return
}
}
// Scoped exactly as before (projectId match only, no workflowId
// fallback — see the loop below) but via an indexed query
// instead of an organization-wide scan.
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
target := &task.Plan.Targets[index]
target.Error = ""
target.Status = "running"
persistErr := persist()
snapshot := *target
mu.Unlock()
if persistErr != nil {
mu.Lock()
if firstErr == nil {
firstErr = persistErr
}
mu.Unlock()
cancel()
return
}
runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot)
mu.Lock()
target = &task.Plan.Targets[index]
if runErr != nil {
target.Status = "failed"
target.Error = runErr.Error()
if firstErr == nil {
firstErr = runErr
}
} else {
target.Status = "done"
}
_ = persist()
mu.Unlock()
}(index)
}
wg.Wait()
return firstErr
}
indicesByKind := map[string][]int{}
nonRuntimePending := false
for index, t := range task.Plan.Targets {
indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index)
if t.Kind != "runtime" && t.Status != "done" {
nonRuntimePending = true
}
}
if e := processBatch(indicesByKind["runtime"]); e != nil {
fail(e)
return
}
// Re-run the phase-transition safety checks (wait for in-flight human
// work, re-verify runtime absence, revalidate references) exactly once
// per execution attempt, right before touching any file/storage target
// — the same point the original sequential loop ran them at.
if nonRuntimePending {
deadline := time.Now().Add(150 * time.Second)
for {
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
if e != nil {
fail(e)
return
}
known := map[string]bool{}
busy := false
for _, id := range task.Plan.RunIDs {
known[id] = true
run, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if run != nil && run.ExecutionLeaseOwner != "" {
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
if e != nil || until.After(time.Now()) {
busy = true
}
}
}
for _, r := range fresh {
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
for i := range items {
item := &items[i]
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
fail(e)
return
}
w := mapValue(item.Subject.Payload["sliceWorkflow"])
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
busy = true
}
}
if !busy {
break
}
if time.Now().After(deadline) {
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
return
}
select {
case <-ctx.Done():
fail(ctx.Err())
return
case <-time.After(2 * time.Second):
}
}
// Catch work admitted immediately before the project lock.
for _, step := range task.Plan.Targets {
if step.Kind == "runtime" {
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
fail(e)
return
}
}
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
fail(e)
return
}
runtimeDone = true
}
target.Error = ""
target.Status = "running"
if e := persist(); e != nil {
cancel()
return
}
if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil {
target.Status = "failed"
target.Error = e.Error()
// Scoped exactly as before (projectId match only, no workflowId
// fallback — see the loop below) but via an indexed query
// instead of an organization-wide scan.
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
if e != nil {
fail(e)
return
}
target.Status = "done"
if e := persist(); e != nil {
cancel()
known := map[string]bool{}
for _, id := range task.Plan.RunIDs {
known[id] = true
}
for _, r := range fresh {
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
return
}
}
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
fail(e)
return
}
}
for _, kind := range []string{"workspace", "bos", "records"} {
if e := processBatch(indicesByKind[kind]); e != nil {
fail(e)
return
}
}

View File

@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) {
t.Fatal("raw data removed")
}
}
// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the
// processBatch concurrency added to executeProjectDeletion: with many runs
// (many runtime + workspace targets processed by a worker pool instead of
// one at a time), the task must still end up "completed" with every target
// "done" and every run's workspace directory actually removed — i.e. the
// concurrent rewrite produces the same end state as the old sequential
// loop, just faster.
func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"})
if e != nil {
t.Fatal(e)
}
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
if e != nil {
t.Fatal(e)
}
s := &server{store: db, workspaceMountPath: t.TempDir()}
const runCount = 30
dirs := make([]string, 0, runCount)
for i := 0; i < runCount; i++ {
runID := fmt.Sprintf("run-concurrent-fixture-%02d", i)
if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
t.Fatal(e)
}
dir, e := safeRunWorkspace(s.workspaceMountPath, runID)
if e != nil {
t.Fatal(e)
}
if e = os.MkdirAll(dir, 0700); e != nil {
t.Fatal(e)
}
dirs = append(dirs, dir)
}
plan, e := s.buildProjectDeletionPlan(ctx, p)
if e != nil {
t.Fatal(e)
}
if len(plan.Blockers) > 0 {
t.Fatal(plan.Blockers)
}
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
if e != nil {
t.Fatal(e)
}
old := projectDeletionRuntimeFactory
defer func() { projectDeletionRuntimeFactory = old }()
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
}
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil || !ok {
t.Fatal(e)
}
s.executeProjectDeletion(*claimed, "owner")
finished, _ := db.GetProjectDeletion(ctx, task.ID)
if finished.Status != "completed" {
t.Fatalf("%+v", finished)
}
for _, target := range finished.Plan.Targets {
if target.Status != "done" {
t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target)
}
}
for _, dir := range dirs {
if _, e = os.Stat(dir); !os.IsNotExist(e) {
t.Fatalf("workspace %s not removed", dir)
}
}
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
t.Fatal("project remains")
}
}

View File

@ -8,6 +8,15 @@ import (
)
func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Request, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, batch *domain.HumanBatchRecord, payload annotationReviewInput) {
returnAuthor := ""
if payload.Decision == domain.AnnotationReviewRejected {
var err error
returnAuthor, err = s.reworkAuthor(r.Context(), item)
if err != nil {
renderError(w, http.StatusInternalServerError, err)
return
}
}
op := operationFromRequest(r)
now := op.CreatedAt
beforeTask, beforeItem := task, item
@ -26,6 +35,10 @@ func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Reques
item.ReviewRound++
item.CurrentStageID = "annotation_combined"
item.Status = domain.HumanItemRework
item.AssignedUserID, task.AssignedUserID = returnAuthor, returnAuthor
if returnAuthor != "" {
item.ClaimedAt = now
}
if item.ReviewRound > item.MaxReworkRounds {
item.Status = domain.HumanItemEscalated
task.Status = domain.AnnotationTaskStatusRejected

View File

@ -0,0 +1,41 @@
package api
import (
"context"
"droboticflow/internal/domain"
"strings"
)
// Resolve before recording a decision: transient lookup failures must not silently
// turn a return-to-author decision into a public-pool assignment.
func (s *server) reworkAuthor(ctx context.Context, item domain.HumanItemRecord) (string, error) {
if item.LatestAuthorUserID == "" || item.ReviewRound+1 > item.MaxReworkRounds {
return "", nil
}
user, err := s.store.GetUser(ctx, item.LatestAuthorUserID)
if err != nil {
return "", err
}
if user == nil || user.OrganizationID != item.OrganizationID || user.Status != domain.UserStatusActive || user.Role != domain.UserRoleAnnotator || user.SupplierID == "" {
return "", nil
}
if item.SupplierID != "" && item.SupplierID != user.SupplierID {
return "", nil
}
supplier, err := s.store.GetSupplier(ctx, user.SupplierID)
if err != nil {
return "", err
}
if supplier == nil || supplier.OrganizationID != item.OrganizationID || !strings.EqualFold(supplier.Status, "active") {
return "", nil
}
members, err := s.annotationMembershipsForUser(ctx, item.OrganizationID, item.ProjectID, user.ID)
if err != nil {
return "", err
}
task := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, ProcessingScope: item.ProcessingScope}
if !membershipHasRole(members, task, domain.AnnotationRoleAnnotator) {
return "", nil
}
return user.ID, nil
}

View File

@ -0,0 +1,85 @@
package api
import (
"droboticflow/internal/domain"
"encoding/json"
"net/http"
"testing"
)
func TestQualityRejectionReturnsToEligibleAuthor(t *testing.T) {
for _, durable := range []bool{false, true} {
for _, scenario := range []string{"eligible", "disabled", "wrong-supplier", "no-membership", "supplier-inactive", "no-author", "escalated"} {
t.Run(scenario+map[bool]string{true: "-durable", false: "-legacy"}[durable], func(t *testing.T) {
f := decisionFixture(t)
f.item.LatestAuthorUserID = f.annotator.ID
f.item.SupplierID = f.annotator.SupplierID
f.task.SupplierID = f.annotator.SupplierID
f.item.MaxReworkRounds = 5
switch scenario {
case "disabled":
f.annotator.Status = domain.UserStatusDisabled
f.store.UpsertUser(f.ctx, f.annotator)
case "wrong-supplier":
f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: "other", OrganizationID: f.item.OrganizationID, Name: "Other", Status: "active"})
f.annotator.SupplierID = "other"
f.annotator.ID = "different-supplier-author"
f.item.LatestAuthorUserID = f.annotator.ID
f.store.UpsertUser(f.ctx, f.annotator)
case "no-membership":
f.store.DeleteProjectMember(f.ctx, "member-"+f.annotator.ID)
case "supplier-inactive":
f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: f.annotator.SupplierID, OrganizationID: f.item.OrganizationID, Name: "Inactive", Status: "inactive"})
case "no-author":
f.item.LatestAuthorUserID = ""
case "escalated":
f.item.MaxReworkRounds = 1
}
f.store.UpsertHumanItem(f.ctx, f.item)
f.store.UpsertAnnotationTask(f.ctx, f.task)
input := map[string]any{"stage": "quality", "decision": "rejected", "reasonCodes": []string{"manual_review_reject"}, "comment": "请修正", "problemSliceIds": []string{"left"}}
endpoint := "/api/plugins/annotation-qa/tasks/" + f.task.ID + "/reviews"
var code int
var body string
if durable {
version, headers := openReviewDraftSession(t, f)
input["operationId"] = "return-author"
input["draftVersion"] = 0
input["resultVersion"] = version
r := reviewSessionRequest(f, http.MethodPost, endpoint, f.reviewer, input, headers)
code = r.Code
body = r.Body.String()
} else {
r := f.request(http.MethodPost, endpoint, f.reviewer, input)
code = r.Code
body = r.Body.String()
}
if code != 201 {
t.Fatalf("review %d %s", code, body)
}
item, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
task, _ := f.store.GetAnnotationTask(f.ctx, f.task.ID)
expected := ""
if scenario == "eligible" {
expected = f.annotator.ID
}
if item.AssignedUserID != expected || task.AssignedUserID != expected {
t.Fatalf("return owner item=%q task=%q want=%q", item.AssignedUserID, task.AssignedUserID, expected)
}
if item.CurrentRevision != f.item.CurrentRevision {
t.Fatal("annotation revision changed")
}
if expected != "" {
r := f.request(http.MethodGet, "/api/human-items?pool=mine&projectId="+item.ProjectID, f.annotator, nil)
var p struct {
Items []domain.HumanItemRecord `json:"items"`
}
json.Unmarshal(r.Body.Bytes(), &p)
if len(p.Items) != 1 {
t.Fatalf("returned task missing in author's pool: %s", r.Body.String())
}
}
})
}
}
}

View File

@ -10449,7 +10449,7 @@ func TestObjectStorageContentSupportsRangeAndHeadWithoutReadingWholeObject(t *te
if item == nil || item.ID != connection.ID {
t.Fatalf("unexpected storage connection: %#v", item)
}
return fakeBrowser, func() {}, nil
return &unsignedVideoBrowser{fakeBrowser}, func() {}, nil
}
defer func() { storageConnectionBrowserResolver = originalResolver }()

View File

@ -0,0 +1,86 @@
package api
import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// A backend without presigning still supports the existing Range proxy.
type unsignedVideoBrowser struct{ *fakeObjectStorageBrowser }
func (b *unsignedVideoBrowser) BuildPreviewURL(context.Context, string, string) (string, error) {
return "", nil
}
func TestVideoPlaybackRedirectsRangesWithoutProxyingBytes(t *testing.T) {
ctx := context.Background()
data := store.NewMemoryStore()
connection, err := data.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "video-test", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test",
Bucket: "videos", AccessKeyID: "test-ak", SecretAccessKey: "test-sk",
})
if err != nil {
t.Fatal(err)
}
browser := newFakeObjectStorageBrowser()
key := "videos|capture.mp4"
browser.objects[key] = domain.ObjectStorageObjectSummary{
Bucket: "videos", Key: "capture.mp4", Name: "capture.mp4", Kind: "video",
SizeBytes: 10, ContentType: "video/mp4",
}
browser.content[key] = []byte("0123456789")
old := storageConnectionBrowserResolver
defer func() { storageConnectionBrowserResolver = old }()
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
return browser, nil, nil
}
handler := NewHandler(data, nil)
path := "/api/object-storage/content?connectionId=" + url.QueryEscape(connection.ID) + "&bucket=videos&key=capture.mp4"
for _, byteRange := range []string{"bytes=0-", "bytes=2-5", "bytes=-4", ""} {
request := httptest.NewRequest(http.MethodGet, path, nil)
request.Header.Set("Range", byteRange)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusTemporaryRedirect {
t.Fatalf("range %q: status %d, want 307", byteRange, response.Code)
}
if response.Header().Get("Location") != "https://example.com/preview/videos/capture.mp4" {
t.Fatal("expected presigned storage target")
}
if response.Header().Get("Cache-Control") != "private, no-store" {
t.Fatal("signed redirects must not be cached")
}
}
if browser.readCalls[key] != 0 || browser.readRangeCalls[key] != 0 {
t.Fatal("API must not transfer any video bytes")
}
// Invalid ranges remain rejected locally; metadata-only reads remain HEAD.
for _, tc := range []struct {
method, byteRange string
status int
}{
{http.MethodGet, "bytes=20-", 416}, {http.MethodHead, "", 200},
} {
request := httptest.NewRequest(tc.method, path, nil)
request.Header.Set("Range", tc.byteRange)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != tc.status || response.Header().Get("Location") != "" {
t.Fatalf("method %s range %q: status %d", tc.method, tc.byteRange, response.Code)
}
}
// The original authorization gate is still in front of URL issuance.
t.Setenv("DROBOTICFLOW_REQUIRE_SESSION_AUTH", "true")
authenticatedHandler := NewHandler(data, nil)
response := httptest.NewRecorder()
authenticatedHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil))
if response.Code != http.StatusUnauthorized || response.Header().Get("Location") != "" {
t.Fatalf("anonymous request leaked video URL: status %d", response.Code)
}
}

View File

@ -872,6 +872,12 @@ type RunRecord struct {
Error string `json:"error,omitempty" bson:"error,omitempty"`
CreatedAt string `json:"createdAt" bson:"createdAt"`
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
// ArchivedAt marks a terminal run excluded from org-wide scans (e.g. the
// deletion-safety reference check) once it's old enough that its data no
// longer needs to participate in day-to-day queries. The record itself is
// untouched — archiving never deletes it; only deleting the run's own
// project does that.
ArchivedAt string `json:"archivedAt,omitempty" bson:"archivedAt,omitempty"`
}
type RunSummary struct {

View File

@ -84,7 +84,7 @@ func (store *MongoStore) updateAdministrator(ctx context.Context, item domain.Us
_, e = store.users.UpdateOne(sc, immutableSupplierFilter(item.ID, item.SupplierID), bson.M{"$set": fields})
return nil, e
})
if mongo.IsDuplicateKeyError(err) {
if isPhoneUniqueDuplicateKeyError(err) {
return ErrDuplicateUserPhone
}
return err

View File

@ -5,6 +5,8 @@ import (
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
@ -14,22 +16,60 @@ import (
var ErrAccountMutationBusy = errors.New("account mutation is busy or requires recovery")
// A durable, per-resource mutex also works on standalone MongoDB. It deliberately
// has no expiring lease: a paused process must never resume after another writer
// has stolen its lock. After a process crash or ambiguous network result, an
// operator must verify that owner has stopped and inspect the affected resource
// before clearing that exact owner's lock. Normal exits release their own lock.
// accountMutationLockLeaseDuration bounds how long a held lock survives its
// owner going silent (crash, pod replacement, OOM) before another process
// may reclaim it. Renewed well before expiry by a background ticker while
// the guarded operation is still running (see withAccountMutationLock), so
// a live owner's lease never lapses out from under it — only a dead one's
// does. Configurable because "how long is safe to wait before auto-
// reclaiming" is an operational judgment call, not a constant.
func accountMutationLockLeaseDuration() time.Duration {
const fallback = 2 * time.Minute
raw := strings.TrimSpace(os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS"))
if raw == "" {
return fallback
}
seconds, err := strconv.Atoi(raw)
if err != nil || seconds <= 0 {
return fallback
}
return time.Duration(seconds) * time.Second
}
// A durable, per-resource mutex also works on standalone MongoDB. It carries
// an expiring lease (accountMutationLockLeaseDuration) renewed periodically
// by a background ticker for as long as the guarded operation keeps running,
// so a live owner's lock is never reclaimed out from under it — only a lock
// whose owner stopped renewing (crashed, was replaced, or otherwise went
// silent) becomes reclaimable, once its lease lapses. Reclaiming a lapsed
// lease always assigns a fresh owner id, so the original holder's own
// release (guarded by "owner": owner) can never match afterward, and never
// touches the new holder's lock — the fresh id already acts as the fencing
// token a naive TTL-lock would be missing. A crash or ambiguous network
// result during the guarded write is still surfaced for the caller to
// verify, exactly as before; only the "must wait for an operator to notice
// and clear it by hand" half of the old design is gone.
func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, operation func(context.Context) (any, error)) (result any, err error) {
col := s.db.Collection("accountMutationLocks")
_, e := col.InsertOne(ctx, bson.M{"_id": key, "locked": false})
if e != nil && !mongo.IsDuplicateKeyError(e) {
return nil, e
}
leaseDuration := accountMutationLockLeaseDuration()
owner := domain.NewID("mutation")
waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
for {
r, e := col.UpdateOne(waitCtx, bson.M{"_id": key, "locked": false}, bson.M{"$set": bson.M{"locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"), "startedAt": time.Now().UTC()}})
now := time.Now().UTC()
filter := bson.M{"_id": key, "$or": bson.A{
bson.M{"locked": false},
bson.M{"locked": true, "leaseExpiresAt": bson.M{"$lt": now}},
}}
update := bson.M{"$set": bson.M{
"locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"),
"startedAt": now, "leaseExpiresAt": now.Add(leaseDuration),
}}
r, e := col.UpdateOne(waitCtx, filter, update)
if e != nil {
return nil, e
}
@ -42,14 +82,52 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op
case <-time.After(25 * time.Millisecond):
}
}
// Renew well inside the lease window (a fixed fraction of it) so normal
// scheduling jitter never lets a live owner's lease lapse; only an
// owner that stops renewing entirely — because it's gone — loses the
// lock. Mirrors the project-deletion task lease's own renewal ticker.
renewCtx, cancelRenew := context.WithCancel(context.Background())
renewalDone := make(chan struct{})
go func() {
defer close(renewalDone)
ticker := time.NewTicker(leaseDuration / 4)
defer ticker.Stop()
for {
select {
case <-renewCtx.Done():
return
case <-ticker.C:
renewalCtx, renewalCancel := context.WithTimeout(context.Background(), 5*time.Second)
r, e := col.UpdateOne(renewalCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"leaseExpiresAt": time.Now().UTC().Add(leaseDuration)}})
renewalCancel()
if e != nil {
// A transient error (network blip, timeout) — not proof
// the lock was actually lost. Keep the lease as-is and
// retry on the next tick; giving up here on a live owner
// would reintroduce the exact "two concurrent writers"
// race the lease is meant to prevent.
continue
}
if r.MatchedCount != 1 {
// The document no longer matches our owner: someone else
// genuinely reclaimed it after our lease lapsed. Stop
// renewing; the release below will correctly no-op or
// surface the conflict.
return
}
}
}
}()
defer func() {
cancelRenew()
<-renewalDone
// Do not unlock an uncertain write and admit a conflicting operation.
if ctx.Err() != nil || mongo.IsNetworkError(err) || mongo.IsTimeout(err) {
return
}
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer releaseCancel()
r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": ""}})
r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": "", "leaseExpiresAt": ""}})
if e != nil || r.MatchedCount != 1 {
err = fmt.Errorf("%w: operation finished but lock release needs verification", ErrAccountMutationBusy)
}

View File

@ -0,0 +1,145 @@
package store
import (
"context"
"errors"
"os"
"sync/atomic"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// This test deliberately uses MongoDB, not MemoryStore: the lease/reclaim
// behavior is a persisted-document race the in-memory store can't
// reproduce. Found live on Fanyu: a lock whose owning process was replaced
// mid-hold (a rolling worker deploy, or a crash) stayed "locked" forever —
// the original design had no expiring lease by intent, trading availability
// for a hand-verified recovery. This locks in the replacement: an expiring,
// renewed lease that only a genuinely dead owner ever loses.
func TestAccountMutationLockLeaseReclaimAndRenewal(t *testing.T) {
uri := os.Getenv("MONGO_INDEX_TEST_URI")
if uri == "" {
t.Skip("MONGO_INDEX_TEST_URI required for real Mongo lock regression")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(ctx)
db := os.Getenv("MONGO_INDEX_TEST_DB")
if db == "" {
t.Fatal("MONGO_INDEX_TEST_DB required")
}
dbHandle := client.Database(db)
defer dbHandle.Collection("accountMutationLocks").Drop(context.Background())
s := &MongoStore{db: dbHandle}
t.Run("normal acquire and release round-trips and unblocks the next waiter", func(t *testing.T) {
key := "lease-test-normal"
_, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "first", nil })
if err != nil {
t.Fatalf("first acquire: %v", err)
}
var doc bson.M
if err := dbHandle.Collection("accountMutationLocks").FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil {
t.Fatal(err)
}
if doc["locked"] != false {
t.Fatalf("expected lock released after normal completion, got %#v", doc)
}
if _, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "second", nil }); err != nil {
t.Fatalf("second acquire after release: %v", err)
}
})
t.Run("a lease that lapsed because its owner went silent is reclaimable", func(t *testing.T) {
key := "lease-test-lapsed"
col := dbHandle.Collection("accountMutationLocks")
// Simulate a crashed holder: locked, with a lease already in the past.
if _, err := col.InsertOne(ctx, bson.M{
"_id": key, "locked": true, "owner": "mutation-dead-owner", "process": "dead-pod",
"startedAt": time.Now().UTC().Add(-time.Hour), "leaseExpiresAt": time.Now().UTC().Add(-time.Minute),
}); err != nil {
t.Fatal(err)
}
claimed := make(chan struct{})
result, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) {
close(claimed)
return "reclaimed", nil
})
select {
case <-claimed:
default:
t.Fatal("operation never ran — lapsed lease was not reclaimed")
}
if err != nil {
t.Fatalf("reclaim after lapse: %v", err)
}
if result != "reclaimed" {
t.Fatalf("unexpected result: %v", result)
}
var doc bson.M
if err := col.FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil {
t.Fatal(err)
}
if doc["locked"] != false {
t.Fatalf("expected lock released after reclaim completed, got %#v", doc)
}
})
t.Run("a live owner's lease is renewed before it can lapse, blocking a concurrent reclaim attempt", func(t *testing.T) {
key := "lease-test-renewed"
originalLease := os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS")
os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", "1") // renew ticks every ~250ms
defer func() {
if originalLease == "" {
os.Unsetenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS")
} else {
os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", originalLease)
}
}()
holderStarted := make(chan struct{})
holderMayFinish := make(chan struct{})
var reclaimAttempted atomic.Bool
var reclaimErr error
done := make(chan struct{})
go func() {
defer close(done)
_, e := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) {
close(holderStarted)
<-holderMayFinish
return "held", nil
})
if e != nil {
t.Errorf("long-held operation returned an error: %v", e)
}
}()
<-holderStarted
// Outlive several renewal ticks (lease=1s, renew every ~250ms) while
// still holding the lock, then attempt a short-timeout reclaim: it
// must fail, proving the live holder's lease never lapsed.
time.Sleep(2500 * time.Millisecond)
reclaimCtx, reclaimCancel := context.WithTimeout(ctx, 500*time.Millisecond)
_, reclaimErr = s.withAccountMutationLock(reclaimCtx, key, func(context.Context) (any, error) { return "stolen", nil })
reclaimCancel()
reclaimAttempted.Store(true)
close(holderMayFinish)
<-done
if !reclaimAttempted.Load() {
t.Fatal("reclaim was never attempted")
}
if !errors.Is(reclaimErr, ErrAccountMutationBusy) {
t.Fatalf("expected the renewed lease to block a concurrent reclaim, got: %v", reclaimErr)
}
})
}

View File

@ -0,0 +1,88 @@
package store
import (
"context"
"fmt"
"os"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex reproduces the exact
// bug found live on Fanyu: a users collection that still carries a stale,
// pre-phone-migration email_unique index (this codebase no longer declares
// it, but an older database can still have it — see EnsureIndexes'
// dropObsoleteIndexes call). A duplicate-key error from that unrelated index
// must not be misreported as a phone conflict, or every account created
// after the first one missing an email field fails with a "phone must be
// unique" error that has nothing to do with the phone number entered.
func TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex(t *testing.T) {
uri := os.Getenv("MONGO_INDEX_TEST_URI")
if uri == "" {
t.Skip("MONGO_INDEX_TEST_URI required for real Mongo index regression")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(ctx)
db := os.Getenv("MONGO_INDEX_TEST_DB")
if db == "" {
t.Fatal("MONGO_INDEX_TEST_DB required")
}
c := client.Database(db).Collection(fmt.Sprintf("users_phone_dup_test_%d", time.Now().UnixNano()))
defer c.Drop(context.Background())
// A pre-migration-shaped users collection: phone_unique (current) plus a
// leftover, non-sparse email_unique (obsolete, but not yet dropped).
if _, err := c.Indexes().CreateMany(ctx, []mongo.IndexModel{
uniqueIndex("phone_unique", bson.D{{Key: "phone", Value: 1}}),
uniqueIndex("email_unique", bson.D{{Key: "email", Value: 1}}),
}); err != nil {
t.Fatal(err)
}
// First account has no email field — occupies email_unique's one null slot.
if _, err := c.InsertOne(ctx, bson.M{"id": "user-a", "phone": "13800000001"}); err != nil {
t.Fatal(err)
}
// A second account, also missing email but with a genuinely distinct
// phone, collides only with the obsolete email index — must NOT be
// classified as a phone conflict.
_, err = c.InsertOne(ctx, bson.M{"id": "user-b", "phone": "13800000002"})
if !mongo.IsDuplicateKeyError(err) {
t.Fatalf("expected a duplicate-key error from the stale email index, got: %v", err)
}
if isPhoneUniqueDuplicateKeyError(err) {
t.Fatalf("email_unique conflict misclassified as a phone conflict: %v", err)
}
// A third account reusing user-a's actual phone must still be correctly
// classified as a real phone conflict.
_, err = c.InsertOne(ctx, bson.M{"id": "user-c", "phone": "13800000001", "email": "user-c@example.com"})
if !mongo.IsDuplicateKeyError(err) {
t.Fatalf("expected a duplicate-key error from the phone index, got: %v", err)
}
if !isPhoneUniqueDuplicateKeyError(err) {
t.Fatalf("real phone_unique conflict not recognized: %v", err)
}
if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil {
t.Fatal(err)
}
if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil {
t.Fatalf("dropping an already-absent index must be a no-op: %v", err)
}
// With the obsolete index gone, a second account missing email succeeds.
if _, err := c.InsertOne(ctx, bson.M{"id": "user-d", "phone": "13800000003"}); err != nil {
t.Fatalf("account creation still blocked after dropping the obsolete index: %v", err)
}
}

View File

@ -1281,23 +1281,34 @@ func (store *MemoryStore) ListRunArtifacts(ctx context.Context, status string) (
return items, nil
}
// Excludes archived runs, matching MongoStore's ListRunArtifactsByOrganization
// (see ArchiveTerminalRuns) — this is the org-wide scan behind
// validateDeletionReferences' safety check.
func (store *MemoryStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
items, err := store.ListRunsByOrganization(ctx, organizationID, status)
if err != nil {
return nil, err
}
hydrateRunArtifactsWhenNeeded(items)
return items, nil
kept := make([]domain.RunRecord, 0, len(items))
for _, run := range items {
if run.ArchivedAt == "" {
kept = append(kept, run)
}
}
hydrateRunArtifactsWhenNeeded(kept)
return kept, nil
}
// ListRunArtifactsForProject mirrors the Mongo implementation's scope: runs
// whose projectId matches projectID directly, or whose workflowId is in
// workflowIDs.
// workflowIDs. Deliberately includes archived runs — a project's own
// deletion must still clean up every run it ever had, archived or not.
func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) {
all, err := store.ListRunArtifactsByOrganization(ctx, organizationID, "")
all, err := store.ListRunsByOrganization(ctx, organizationID, "")
if err != nil {
return nil, err
}
hydrateRunArtifactsWhenNeeded(all)
workflowSet := make(map[string]bool, len(workflowIDs))
for _, id := range workflowIDs {
workflowSet[id] = true
@ -1311,6 +1322,29 @@ func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organi
return items, nil
}
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
func (store *MemoryStore) ArchiveTerminalRuns(_ context.Context, olderThan time.Time) (int, error) {
store.mu.Lock()
defer store.mu.Unlock()
threshold := olderThan.UTC().Format(time.RFC3339Nano)
archived := 0
for id, run := range store.runs {
if run.ArchivedAt != "" {
continue
}
if run.Status != domain.RunStatusCompleted && run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled {
continue
}
if run.EndedAt == "" || run.EndedAt >= threshold {
continue
}
run.ArchivedAt = nowISO()
store.runs[id] = run
archived++
}
return archived, nil
}
func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunStatus]int, error) {
store.mu.Lock()
defer store.mu.Unlock()

View File

@ -1211,3 +1211,102 @@ func TestListRunArtifactsForProjectScopesByProjectOrWorkflow(t *testing.T) {
t.Fatalf("scope leaked unrelated runs: %#v", got)
}
}
func TestArchiveTerminalRunsOnlyArchivesOldTerminalUnarchivedRuns(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano)
recent := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339Nano)
threshold := time.Now().Add(-30 * 24 * time.Hour)
runs := []domain.RunRecord{
{ID: "run-old-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old},
{ID: "run-old-failed", OrganizationID: "org-1", Status: domain.RunStatusFailed, EndedAt: old},
{ID: "run-old-cancelled", OrganizationID: "org-1", Status: domain.RunStatusCancelled, EndedAt: old},
{ID: "run-recent-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: recent},
{ID: "run-old-running", OrganizationID: "org-1", Status: domain.RunStatusRunning, EndedAt: ""},
{ID: "run-already-archived", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old},
}
for _, run := range runs {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run %s: %v", run.ID, err)
}
}
archived, err := dataStore.ArchiveTerminalRuns(ctx, threshold)
if err != nil {
t.Fatalf("archive terminal runs: %v", err)
}
if archived != 3 {
t.Fatalf("expected 3 runs archived, got %d", archived)
}
for _, id := range []string{"run-old-completed", "run-old-failed", "run-old-cancelled"} {
run, err := dataStore.GetRun(ctx, id)
if err != nil {
t.Fatalf("get run %s: %v", id, err)
}
if run.ArchivedAt == "" {
t.Fatalf("expected %s to be archived", id)
}
}
for _, id := range []string{"run-recent-completed", "run-old-running"} {
run, err := dataStore.GetRun(ctx, id)
if err != nil {
t.Fatalf("get run %s: %v", id, err)
}
if run.ArchivedAt != "" {
t.Fatalf("expected %s to remain unarchived, got %s", id, run.ArchivedAt)
}
}
// Re-running must not re-archive (and must not double count) an already
// archived run or one that just got archived.
archivedAgain, err := dataStore.ArchiveTerminalRuns(ctx, threshold)
if err != nil {
t.Fatalf("archive terminal runs (second pass): %v", err)
}
if archivedAgain != 0 {
t.Fatalf("expected second pass to archive nothing, archived %d", archivedAgain)
}
}
func TestListRunArtifactsByOrganizationExcludesArchivedRunsButForProjectStillIncludesThem(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano)
runs := []domain.RunRecord{
{ID: "run-active", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old},
{ID: "run-archived", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old},
}
for _, run := range runs {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run %s: %v", run.ID, err)
}
}
orgItems, err := dataStore.ListRunArtifactsByOrganization(ctx, "org-1", "")
if err != nil {
t.Fatalf("list run artifacts by organization: %v", err)
}
if len(orgItems) != 1 || orgItems[0].ID != "run-active" {
t.Fatalf("expected only the unarchived run from the org-wide scan, got %#v", orgItems)
}
// A project's own deletion planning must still see its archived runs —
// archiving must never exempt a run from being cleaned up by its own
// project's deletion.
projectItems, err := dataStore.ListRunArtifactsForProject(ctx, "org-1", "project-a", nil)
if err != nil {
t.Fatalf("list run artifacts for project: %v", err)
}
got := map[string]bool{}
for _, item := range projectItems {
got[item.ID] = true
}
if !got["run-active"] || !got["run-archived"] {
t.Fatalf("expected project scope to include both runs regardless of archive state, got %#v", got)
}
}

View File

@ -1073,8 +1073,12 @@ func (store *MongoStore) ListRunArtifactsForProject(ctx context.Context, organiz
return items, nil
}
// Excludes archived runs: this is the query behind validateDeletionReferences'
// org-wide safety scan, and an organization's run history is unbounded and
// never otherwise shrinks, so a run old enough to be archived is excluded
// here to keep that scan's cost from growing forever (see ArchiveTerminalRuns).
func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
filter := bson.M{}
filter := bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}}
if status != "" {
filter["status"] = status
}
@ -1098,6 +1102,20 @@ func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, org
return items, nil
}
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) {
filter := bson.M{
"status": bson.M{"$in": bson.A{string(domain.RunStatusCompleted), string(domain.RunStatusFailed), string(domain.RunStatusCancelled)}},
"endedAt": bson.M{"$ne": "", "$lt": olderThan.UTC().Format(time.RFC3339Nano)},
"archivedAt": bson.M{"$in": bson.A{nil, ""}},
}
result, err := store.runs.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"archivedAt": nowISO()}})
if err != nil {
return 0, err
}
return int(result.ModifiedCount), nil
}
func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) {
cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{
{{Key: "$group", Value: bson.M{
@ -2237,12 +2255,21 @@ func (store *MongoStore) upsertUser(ctx context.Context, item domain.UserRecord,
return ErrUserMutationConflict
}
}
if mongo.IsDuplicateKeyError(err) {
if isPhoneUniqueDuplicateKeyError(err) {
return ErrDuplicateUserPhone
}
return err
}
// isPhoneUniqueDuplicateKeyError reports whether err is a MongoDB duplicate-key
// error specifically from the users collection's phone_unique index, not just
// any unique-index conflict on that collection — a stale or unrelated index
// (e.g. a pre-migration email_unique index left over in an older database)
// would otherwise be misreported to the caller as "phone must be unique".
func isPhoneUniqueDuplicateKeyError(err error) bool {
return mongo.IsDuplicateKeyError(err) && strings.Contains(err.Error(), "phone_unique")
}
func (store *MongoStore) CreateSession(ctx context.Context, item domain.SessionRecord) (*domain.SessionRecord, error) {
if strings.TrimSpace(item.ID) == "" {
item.ID = domain.NewID("session")

View File

@ -2,6 +2,7 @@ package store
import (
"context"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
@ -118,6 +119,9 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
index("status_updatedAt", bson.D{{Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}),
index("status_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}),
index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}),
// Backs ArchiveTerminalRuns' periodic sweep for terminal, old,
// not-yet-archived runs.
index("status_endedAt", bson.D{{Key: "status", Value: 1}, {Key: "endedAt", Value: 1}}),
index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}),
index("workflowId_taskId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}),
index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}),
@ -378,5 +382,27 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
return err
}
}
return dropObsoleteIndexes(ctx, store.users, "email_unique")
}
// dropObsoleteIndexes removes indexes this codebase no longer declares. A
// database created before the accounts moved from email to phone as the
// sole identifier can still carry a non-sparse email_unique index (this
// function never creates it — only pre-migration deployments have it); left
// in place, the first account created without an email field occupies that
// index's one allowed null slot, and every account created after that
// mislabels as "phone must be unique" since the caller only checks whether
// *some* unique index on the collection conflicted. Missing-index errors
// are ignored since that's the desired end state.
func dropObsoleteIndexes(ctx context.Context, collection *mongo.Collection, names ...string) error {
for _, name := range names {
if _, err := collection.Indexes().DropOne(ctx, name); err != nil {
var cmdErr mongo.CommandError
if errors.As(err, &cmdErr) && cmdErr.Code == 27 { // IndexNotFound
continue
}
return err
}
}
return nil
}

View File

@ -104,6 +104,14 @@ type Store interface {
// every run in the organization (see account-phone-login-era perf fix:
// large orgs made the unscoped org-wide scan hang for minutes).
ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error)
// ArchiveTerminalRuns sets ArchivedAt on every not-yet-archived run whose
// status is completed/failed/cancelled and whose EndedAt is older than
// olderThan. Archiving never deletes the record — it only excludes it
// from the org-wide scans (see ListRunArtifactsByOrganization) that
// would otherwise keep growing as an organization's run history grows
// unbounded; a run's data is only actually removed when its project is
// deleted. Returns the number of runs archived.
ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error)
CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error)
ListRunSummaries(context.Context, string) ([]domain.RunSummary, error)
ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error)

View File

@ -99,6 +99,9 @@ type Runtime struct {
lastOrphanCheckNano atomic.Int64
pendingFailureAfter time.Duration
missingMaxRecoveries int
runArchiveMinimumAge time.Duration
runArchiveInterval time.Duration
lastRunArchiveNano atomic.Int64
}
type ProcessOnceStatus string
@ -203,6 +206,8 @@ func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName str
orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute),
pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute),
missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3),
runArchiveMinimumAge: envDurationSeconds("WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS", 30*24*time.Hour),
runArchiveInterval: envDurationSeconds("WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS", time.Hour),
}, nil
}
@ -1076,6 +1081,39 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors)
}
// archiveTerminalRuns periodically marks old, terminal runs as archived (see
// domain.RunRecord.ArchivedAt) so an organization's ever-growing run history
// doesn't keep inflating the cost of org-wide scans like
// validateDeletionReferences' safety check. Throttled the same way
// reconcileOrphanedRuntimes is, since this only needs to run occasionally,
// not on every poll tick.
func (runtime *Runtime) archiveTerminalRuns(ctx context.Context) {
now := time.Now().UTC()
interval := runtime.runArchiveInterval
if interval <= 0 {
interval = time.Hour
}
lastCheck := runtime.lastRunArchiveNano.Load()
if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval {
return
}
if !runtime.lastRunArchiveNano.CompareAndSwap(lastCheck, now.UnixNano()) {
return
}
minimumAge := runtime.runArchiveMinimumAge
if minimumAge <= 0 {
minimumAge = 30 * 24 * time.Hour
}
archived, err := runtime.store.ArchiveTerminalRuns(ctx, now.Add(-minimumAge))
if err != nil {
fmt.Printf("[worker] archive terminal runs failed: %v\n", err)
return
}
if archived > 0 {
fmt.Printf("[worker] archived %d terminal run(s) older than %s\n", archived, minimumAge)
}
}
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
startedAt := time.Now()
var processErr error
@ -1138,6 +1176,7 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
}
runtime.applyWorkflowRuntimePolicies(ctx, workflows)
runtime.reconcileOrphanedRuntimes(ctx)
runtime.archiveTerminalRuns(ctx)
queuedRuns := runRecordsFromSummaries(queuedPage.Items)
selectedRunIDs := selectQueuedRunsForDispatchSnapshot(
queuedRuns,

View File

@ -6,6 +6,8 @@ metadata:
namespace: __NAMESPACE__
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels: {app: droboticflow-annotation-processor}
template:
@ -22,8 +24,8 @@ spec:
command: [/usr/local/bin/droboticflow-api]
env:
- {name: ANNOTATION_PROCESSOR_MODE, value: processor}
- {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "2"}
- {name: ANNOTATION_INDEX_BACKFILL, value: "1"}
- {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "1"}
- {name: ANNOTATION_INDEX_BACKFILL, value: "0"}
- {name: WORKSPACE_MOUNT_PATH, value: "__WORKSPACE_MOUNT_PATH__"}
- {name: GLOBAL_BOS_ENDPOINT, value: "__BOS_ENDPOINT__"}
- {name: ANNOTATION_WORKSPACE_CONNECTION_ID, value: "__ANNOTATION_WORKSPACE_CONNECTION_ID__"}

View File

@ -14,3 +14,6 @@ export WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
export WORKSPACE_BOS_BUCKET="${WORKSPACE_BOS_BUCKET:-drobodata-fanyu}"
export ANNOTATION_WORKSPACE_CONNECTION_ID="${ANNOTATION_WORKSPACE_CONNECTION_ID:-storage-d0a49fe4b9881c324f0640ab62359603}"
export ANNOTATION_WORKSPACE_BUCKET="${ANNOTATION_WORKSPACE_BUCKET:-$WORKSPACE_BOS_BUCKET}"
# Keep heavy slice materialization out of the web API.
export ANNOTATION_PROCESSOR_ENABLED="${ANNOTATION_PROCESSOR_ENABLED:-true}"

View File

@ -379,8 +379,6 @@ export function CaptureAnnotationWorkbench({
const [playbackRate, setPlaybackRate] = useState(1);
const [showLeft, setShowLeft] = useState(initialCameraVisibility?.left ?? true);
const [showRight, setShowRight] = useState(initialCameraVisibility?.right ?? Boolean(rightVideoSource));
const effectiveShowLeft = focusedLayout || showLeft;
const effectiveShowRight = focusedLayout ? Boolean(rightVideoSource) : showRight;
const cameraStorageKey = cameraPreferenceKey
? `${isQualityReview || providedReviewCaption ? "review" : "annotation"}-camera-default:${cameraPreferenceKey}`
: "";
@ -394,6 +392,7 @@ export function CaptureAnnotationWorkbench({
};
const [defaultCamera, setDefaultCamera] = useState(readCameraDefault);
const initialInspection = () => {
if (!rightVideoSource) return "left" as const;
if (providedReviewCaption && initialCameraVisibility)
return initialCameraVisibility.left && initialCameraVisibility.right
? null
@ -404,7 +403,11 @@ export function CaptureAnnotationWorkbench({
return preference === "both" ? null : (preference as "left" | "right");
};
const [inspectionCamera, setInspectionCamera] = useState<"left" | "right" | null>(initialInspection);
const effectiveShowLeft = focusedLayout ? inspectionCamera !== "right" || !rightVideoSource : showLeft;
const effectiveShowRight = Boolean(rightVideoSource) && (focusedLayout ? inspectionCamera !== "left" : showRight);
const inspectCamera = (camera: "left" | "right" | null) => {
if (camera === inspectionCamera) return;
transport.prepareCameraChange();
setInspectionCamera(camera);
if (providedReviewCaption) onCameraVisibilityChange?.({ left: camera !== "right", right: camera !== "left" });
};
@ -2650,7 +2653,7 @@ export function CaptureAnnotationWorkbench({
<button
type="button"
aria-pressed={inspectionCamera === "left"}
disabled={!effectiveShowLeft}
disabled={!source}
onClick={() => inspectCamera("left")}
>
放大左相机
@ -2659,7 +2662,7 @@ export function CaptureAnnotationWorkbench({
<button
type="button"
aria-pressed={inspectionCamera === "right"}
disabled={!effectiveShowRight}
disabled={!rightSource}
onClick={() => inspectCamera("right")}
>
放大右相机
@ -2944,7 +2947,17 @@ export function CaptureAnnotationWorkbench({
)}
{transport.mediaError && (
<span role="alert" className="text-xs text-rose-300">
{transport.mediaError}
{transport.mediaError}{" "}
<button
type="button"
className="underline"
onClick={() => {
transport.retry();
focusWorkbench();
}}
>
重新加载视频
</button>
</span>
)}
<span className="ml-auto inline-flex items-center gap-3">

View File

@ -34,6 +34,27 @@ export function useCaptureVideoTransport(options: Options) {
const [isPositioning, setIsPositioning] = useState(false);
const [mediaError, setMediaError] = useState("");
const targets = useRef<{ video: HTMLVideoElement; time: number; revision: number }[]>([]);
const retryTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const retryAttempts = useRef({ left: 0, right: 0 });
const reloadSequence = useRef(0);
function cancelRetry() {
if (retryTimer.current !== null) clearTimeout(retryTimer.current);
retryTimer.current = null;
}
function reloadTrack(video: HTMLVideoElement) {
const c = config.current;
const source = video === c.leftRef.current ? c.leftSource : c.rightSource;
if (!source) return;
const url = new URL(source, window.location.href);
// Return to the original authenticated route, never to currentSrc (which
// can be an expired BOS redirect). Do not alter external URL signatures.
if (url.origin === window.location.origin) {
url.searchParams.set("_mediaRetry", `${Date.now()}-${++reloadSequence.current}`);
}
video.src = url.href;
video.load();
}
function visibleTracks() {
const c = config.current;
@ -66,6 +87,10 @@ export function useCaptureVideoTransport(options: Options) {
}
function seekTrack(target: (typeof targets.current)[number]) {
const { video, time } = target;
if (video.error) {
reloadTrack(video);
return;
}
if (video.readyState < 1) return; // loadedmetadata will finish this request.
const next = physicalTarget(video, time);
if (Math.abs(video.currentTime - next) > 0.000001) video.currentTime = next;
@ -77,19 +102,24 @@ export function useCaptureVideoTransport(options: Options) {
({ video, time }) =>
// HTMLMediaElement can quantize to microseconds. This tolerance is
// only for readiness, never for the saved/visible annotation time.
video.seeking || video.readyState < 2 || Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002,
video.error ||
video.seeking ||
video.readyState < 2 ||
Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002,
)
)
return;
const revision = targets.current[0].revision;
targets.current = [];
setMediaError("");
retryAttempts.current = { left: 0, right: 0 };
queue.complete(revision);
}
const [queue] = useState(() =>
createMediaSeekQueue({
dispatch(time: number, revision: number) {
cancelRetry();
setIsPositioning(true);
setMediaError("");
pauseElements();
targets.current = visibleTracks().map(({ video, toMedia }) => ({ video, time: toMedia(time), revision }));
targets.current.forEach(seekTrack);
@ -105,6 +135,7 @@ export function useCaptureVideoTransport(options: Options) {
);
useEffect(
() => () => {
cancelRetry();
playing.current = false;
queue.cancel();
targets.current = [];
@ -165,11 +196,25 @@ export function useCaptureVideoTransport(options: Options) {
function play() {
playing.current = true;
setIsPlaying(true);
if (visibleTracks().some(({ video }) => video.error)) {
retry();
return;
}
if (!queue.isPending() && !scrubbing.current) playElements();
}
function retry() {
cancelRetry();
retryAttempts.current = { left: 0, right: 0 };
queue.cancel();
queue.commit(position.current);
}
// Sources/visibility, not playhead updates, own resource lifetime.
useEffect(() => {
cancelRetry();
retryAttempts.current = { left: 0, right: 0 };
setMediaError("");
queue.cancel();
targets.current = [];
scrubbing.current = false;
@ -191,6 +236,7 @@ export function useCaptureVideoTransport(options: Options) {
}
queue.commit(position.current);
return () => {
cancelRetry();
queue.cancel();
targets.current = [];
};
@ -210,12 +256,28 @@ export function useCaptureVideoTransport(options: Options) {
onLoadedData: settle,
onCanPlay: settle,
onError(event: { currentTarget: HTMLVideoElement }) {
if (!event.currentTarget.getAttribute("src")) return;
if (
!event.currentTarget.getAttribute("src") ||
!visibleTracks().some(({ video }) => video === event.currentTarget)
)
return;
cancelRetry();
queue.cancel();
targets.current = [];
setIsPositioning(false);
setMediaError("视频加载失败,请重新定位或重新打开任务。");
pause();
setMediaError("视频加载失败,当前编辑已保留。");
// Keep play intent and the logical position, so recovery does not
// remount the editor or jump to the beginning of the video.
pauseElements();
if (config.current.enabled && retryAttempts.current[camera] < 2) {
const attempt = ++retryAttempts.current[camera];
retryTimer.current = setTimeout(() => {
retryTimer.current = null;
queue.commit(position.current);
}, attempt * 750);
} else {
pause();
}
},
onPlay(event: { currentTarget: HTMLVideoElement }) {
if (event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current)
@ -224,7 +286,13 @@ export function useCaptureVideoTransport(options: Options) {
setIsPlaying(true);
},
onPause(event: { currentTarget: HTMLVideoElement }) {
if (!event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current)
if (
!event.currentTarget.paused ||
!isMaster(event.currentTarget) ||
queue.isPending() ||
scrubbing.current ||
retryTimer.current !== null
)
return;
pause();
},
@ -257,6 +325,7 @@ export function useCaptureVideoTransport(options: Options) {
pause,
pauseAtFrame,
play,
retry,
isPlaying,
isPositioning,
mediaError,

View File

@ -0,0 +1,44 @@
// Candidate discovery only. The server revalidates explicit selections and
// creates a new revision; viewing or marking an action invalid never merges.
export function invalidSliceConsolidationCandidates(slices, actions) {
const ordered = [...slices].sort((a, b) => a.start - b.start);
return ordered.flatMap((current, index) => {
const left = ordered[index - 1];
const right = ordered[index + 1];
if (
!left ||
!right ||
[left, current, right].some((s) => s.deliveryDisposition === "discard") ||
!current.name?.trim() ||
left.name !== current.name ||
right.name !== current.name ||
left.end !== current.start ||
current.end !== right.start
)
return [];
const inner = actions
.filter((a) => a.end > current.start && a.start < current.end)
.sort((a, b) => a.start - b.start);
let cursor = current.start;
if (
!inner.length ||
!inner.every((a) => {
if (!a.isInvalid || a.start !== cursor || a.end <= a.start || a.end > current.end) return false;
cursor = a.end;
return true;
}) ||
cursor !== current.end
)
return [];
return [
{
id: current.id,
name: current.name,
start: left.start,
end: right.end,
invalidStart: current.start,
invalidEnd: current.end,
},
];
});
}

View File

@ -1,3 +1,4 @@
import { invalidSliceConsolidationCandidates } from "../lib/invalid-slice-consolidation.js";
import { ReviewDecisionDialog } from "../components/ReviewDecisionDialog";
import { useReviewContinuation } from "../hooks/useReviewContinuation";
import { useReviewEntryIntent } from "../hooks/useReviewEntryIntent";
@ -3157,12 +3158,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
setError("请保存所有动作描述或明确设为无效");
return;
}
const candidates = invalidSliceConsolidationCandidates(sliceSegments, segments);
const consolidateInvalidSliceIds: string[] = [];
for (const candidate of candidates) {
if (
window.confirm(
`“${candidate.name}”中 ${candidate.invalidStart.toFixed(3)}–${candidate.invalidEnd.toFixed(3)} 秒的无效动作单独占了一个切片。\n\n是否与前后同名切片合并为 ${candidate.start.toFixed(3)}–${candidate.end.toFixed(3)} 秒的完整切片?全部动作描述、无效标记及时间保持不变。\n确定:合并外层;取消:保留原划分。`,
)
)
consolidateInvalidSliceIds.push(candidate.id);
}
if (
candidates.length &&
!window.confirm(
consolidateInvalidSliceIds.length
? `已选择 ${consolidateInvalidSliceIds.length} 处无损合并。提交后重新生成审核视频,原版本保留。确认提交质检?`
: "保留原切片划分,确认提交质检?",
)
)
return;
const currentTaskId = detail.task.id;
const token = beginAnnotationMutation(currentTaskId);
if (!token) return;
clearEditHistory();
try {
const submission = await annotationOperationsApi.submitCombined(detail.task.id, {
consolidateInvalidSliceIds,
version: detailRef.current?.humanItem?.version ?? detail.humanItem.version,
sliceSegments: applySliceStatuses(
sliceSegments.map((slice) => ({ ...slice, taskLabelId: "", taskLabel: "" })),

View File

@ -178,6 +178,7 @@ export const annotationOperationsApi = {
taskId: string,
payload: {
version: number;
consolidateInvalidSliceIds?: string[];
sliceSegments: SliceSegment[];
sliceOperations?: SliceOperation[];
segments: AnnotationSegment[];

View File

@ -0,0 +1,170 @@
// Local real-component/media regression. Synthetic video only; no API writes.
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import { execFileSync } from "node:child_process";
import { chromium } from "playwright";
const base = process.env.VIDEO_TEST_URL || "http://127.0.0.1:5196";
assert(["127.0.0.1", "localhost"].includes(new URL(base).hostname), "local fixtures only");
const out = process.env.VIDEO_TEST_OUTPUT || "/tmp/presigned-video-tests";
await fs.mkdir(out, { recursive: true });
const file = out + "/sample.mp4";
try {
await fs.access(file);
} catch {
execFileSync("ffmpeg", [
"-hide_banner",
"-loglevel",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=size=240x320:rate=30:duration=12",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
file,
]);
}
const bytes = await fs.readFile(file);
const browser = await chromium.launch({
executablePath:
process.env.PLAYWRIGHT_EXECUTABLE_PATH || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
headless: true,
args: ["--no-proxy-server"],
});
try {
for (const mode of ["annotation", "quality", "acceptance"]) {
const page = await browser.newPage({ viewport: { width: 1413, height: 900 } });
page.setDefaultTimeout(15000);
let fail = false,
failedRequests = 0;
const requests = [],
errors = [];
page.on("pageerror", (e) => errors.push(e.message));
await page.route("**/api/**", async (route) => {
const req = route.request(),
u = new URL(req.url());
assert.equal(req.method(), "GET");
if (!u.pathname.endsWith(".mp4")) return route.fulfill({ json: {} });
requests.push({ path: u.pathname, retry: u.searchParams.get("_mediaRetry") });
if (fail && u.pathname.includes("left")) {
failedRequests++;
return route.abort("failed");
}
const match = /bytes=(\d+)-(\d*)/.exec(req.headers().range || "");
const start = match ? Number(match[1]) : 0,
end = match?.[2] ? Math.min(Number(match[2]), bytes.length - 1) : bytes.length - 1;
return route.fulfill({
status: match ? 206 : 200,
contentType: "video/mp4",
headers: {
"Accept-Ranges": "bytes",
"Content-Length": String(end - start + 1),
...(match ? { "Content-Range": `bytes ${start}-${end}/${bytes.length}` } : {}),
},
body: bytes.subarray(start, end + 1),
});
});
await page.route("**/__video_recovery", (route) =>
route.fulfill({
contentType: "text/html",
body: `<html><body><div id="root"></div><script type="module">
import RefreshRuntime from '/@react-refresh';RefreshRuntime.injectIntoGlobalHook(window);window.$RefreshReg$=()=>{};window.$RefreshSig$=()=>type=>type;window.__vite_plugin_react_preamble_installed__=true;
const React=await import('/node_modules/.vite/deps/react.js').then(m=>m.default||m);const D=await import('/node_modules/.vite/deps/react-dom_client.js');await import('/src/index.css');
const {CaptureAnnotationWorkbench}=await import('/src/components/CaptureAnnotationWorkbench.tsx');
const mode=${JSON.stringify(mode)};
const props={taskId:'synthetic-video',videoSource:'/api/video-test/left.mp4',rightVideoSource:'/api/video-test/right.mp4',browserReadyMedia:true,sourceIsFullVideo:true,focusedLayout:true,clipStart:0,clipEnd:12,slices:[{id:'slice',name:'Synthetic',start:0,end:12}],segments:[{id:'a',start:0,end:12,description:'Original',source:'manual',confidence:1}],labels:[],statuses:{},phase:'annotate',continuousActions:true,frameBoundaries:Array.from({length:361},(_,i)=>i/30),isQualityReview:mode!=='annotation',reviewMode:mode==='acceptance'?'acceptance':'quality',...(mode==='acceptance'?{reviewCaption:{text:'Original'}}:{}),onChange:()=>{throw Error('Unexpected annotation write')}};
(D.createRoot||D.default.createRoot)(document.getElementById('root')).render(React.createElement('div',{className:'annotation-operations-page','data-workspace-compact':true},React.createElement(CaptureAnnotationWorkbench,props)));
</script></body></html>`,
}),
);
await page.goto(base + "/__video_recovery", { waitUntil: "domcontentloaded" });
const bench = page.getByTestId("capture-annotation-workbench"),
left = page.locator('video[data-camera="left"]'),
right = page.locator('video[data-camera="right"]');
const settle = () =>
page.waitForFunction(() => {
const videos = [...document.querySelectorAll("video")].filter((v) => v.getAttribute("src"));
return videos.length && videos.every((v) => v.readyState >= 2 && !v.seeking && !v.error);
});
await bench.waitFor();
await settle();
assert.equal(await right.getAttribute("src"), null, "single-camera view must detach the hidden stream");
assert.equal(
requests.filter((r) => r.path.includes("right")).length,
0,
"hidden camera must make no media request",
);
await bench.getByRole("button", { name: "播放", exact: true }).click();
await page.waitForFunction(() => document.querySelector('video[data-camera="left"]').currentTime > 1);
const beforeSameCamera = await left.evaluate((v) => v.currentTime);
await bench.getByRole("button", { name: "放大左相机", exact: true }).click();
await page.waitForFunction(
(t) => document.querySelector('video[data-camera="left"]').currentTime > t + 0.3,
beforeSameCamera,
);
// One transient read failure recovers automatically and retains play intent.
fail = true;
await left.evaluate((v) => v.load());
await page.waitForFunction(() => !!document.querySelector('video[data-camera="left"]').error);
fail = false;
await settle();
await page.waitForFunction(() => !document.querySelector('video[data-camera="left"]').paused);
failedRequests = 0;
await bench.getByRole("button", { name: "放大右相机", exact: true }).click();
await settle();
assert.equal(await left.getAttribute("src"), null);
assert((await right.evaluate((v) => v.currentTime)) > 0.8, "camera switch preserves logical playback position");
await bench.getByRole("button", { name: "恢复双画面", exact: true }).click();
await settle();
await bench.getByRole("button", { name: "暂停", exact: true }).click();
await bench.getByRole("button", { name: "放大左相机", exact: true }).click();
await settle();
const held = await left.evaluate((v) => v.currentTime);
let input;
if (mode === "annotation") {
await bench.getByTestId("annotation-description-block").first().click();
input = bench.getByRole("textbox", { name: "动作描述", exact: true });
await input.fill("Unsubmitted draft must survive video recovery");
}
const beforeFailure = await left.evaluate((v) => v.currentTime);
fail = true;
await left.evaluate((v) => v.load());
await page.waitForTimeout(4500);
assert(failedRequests <= 3, `automatic retries must be bounded: ${failedRequests}`);
assert(await left.evaluate((v) => !!v.error), "persistent failure remains visible in the media element");
assert(await bench.getByRole("alert").count(), "do not clear the failure before media recovers");
fail = false;
await bench.getByRole("button", { name: "下一帧", exact: true }).click();
await settle();
assert(
requests.some((r) => r.retry),
"retry returns to original authorized URL for a fresh signature",
);
assert(
Math.abs((await left.evaluate((v) => v.currentTime)) - beforeFailure) < 0.15,
"recovery preserves the frame clock",
);
assert.equal(await left.evaluate((v) => v.paused), true, "paused users are not auto-played");
if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery");
// Explicit retry also works after the bounded automatic attempts are exhausted.
fail = true;
failedRequests = 0;
await left.evaluate((v) => v.load());
await page.waitForTimeout(4500);
fail = false;
await bench.getByRole("button", { name: "重新加载视频", exact: true }).click();
await settle();
assert.equal(await bench.getByRole("alert").count(), 0);
if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery");
assert.deepEqual(errors, []);
console.log(JSON.stringify({ mode, passed: true, held, requests: requests.length }));
await page.close();
}
} finally {
await browser.close();
}

View File

@ -67,8 +67,8 @@ test("both playhead sliders use latest-only pointer previews and exact final com
});
test("hidden sources detach and right-only playback maps back to the canonical frame clock", () => {
assert.match(workbench, /const effectiveShowLeft = focusedLayout \|\| showLeft/);
assert.match(workbench, /const effectiveShowRight = focusedLayout \? Boolean\(rightVideoSource\) : showRight/);
// Real focused-camera resource lifetime is exercised by
// tests/browser/video-playback-recovery.mjs (hidden camera issues no request).
assert.match(workbench, /src=\{effectiveShowLeft \? source : undefined\}/);
assert.match(workbench, /src=\{effectiveShowRight \? rightSource : undefined\}/);
assert.match(workbench, /mapCameraBoundaryTime\(rightMediaBoundaries, frameBoundaries, time\)/);

View File

@ -0,0 +1,40 @@
import test from "node:test";
import assert from "node:assert/strict";
import { invalidSliceConsolidationCandidates as candidates } from "../../src/lib/invalid-slice-consolidation.js";
const slices = ["a", "b", "c"].map((id, i) => ({ id, name: "盘点", start: i, end: i + 1 }));
const actions = [{ id: "invalid", start: 1, end: 2, isInvalid: true, description: "无效片段" }];
test("identify invalid-only outer partition without modifying slices or action JSON", () => {
const before = JSON.stringify({ slices, actions });
assert.deepEqual(candidates(slices, actions), [
{ id: "b", name: "盘点", start: 0, end: 3, invalidStart: 1, invalidEnd: 2 },
]);
assert.equal(JSON.stringify({ slices, actions }), before);
});
test("never propose discarded, different-name, gapped or partially invalid partitions", () => {
assert.deepEqual(
candidates(
slices.map((s) => (s.id === "b" ? { ...s, deliveryDisposition: "discard" } : s)),
actions,
),
[],
);
assert.deepEqual(
candidates(
slices.map((s) => (s.id === "c" ? { ...s, name: "清洁" } : s)),
actions,
),
[],
);
assert.deepEqual(
candidates(
slices.map((s) => (s.id === "c" ? { ...s, start: 2.1 } : s)),
actions,
),
[],
);
assert.deepEqual(candidates(slices, [{ ...actions[0], end: 1.5 }]), []);
assert.deepEqual(candidates(slices, [{ ...actions[0], isInvalid: false }]), []);
});
test("invalid inside a single business slice stays inside it", () => {
assert.deepEqual(candidates([{ id: "a", name: "盘点", start: 0, end: 3 }], actions), []);
});