diff --git a/docs/annotation-invalid-slice-consolidation.md b/docs/annotation-invalid-slice-consolidation.md new file mode 100644 index 00000000..a43ea22e --- /dev/null +++ b/docs/annotation-invalid-slice-consolidation.md @@ -0,0 +1,13 @@ +# 无效动作与外层切片的无损整理 + +无效动作属于外层业务切片内部,设置无效不新增外层边界。旧版取消切分仍按既有规则清空合并区间判断;它不是无损整理入口。 + +送检前识别以下候选:连续三个保留切片名称完全相同、首尾精确相接,中间切片由明确无效的动作完整覆盖。用户逐项选择是否合并,最后再次确认提交。取消选择可保留原划分,不对所有同名切片做自动合并。舍弃、名称不同、缺口、部分无效都不匹配。 + +服务器重新验证选择,仅在已有领取权、版本、字典、覆盖与动作合法性校验全部通过后,改变送检版本的外层划分。动作数组不因合并改变:ID、描述、无效判断、起止边界、硬件时间戳均保留。重叠候选合成同一连续区间。操作历史记录原切片快照。 + +后台生成新的媒体和 JSON;不能把旧短片直接冒充合并后视频。JSON 中相对于切片起点的时间随新的切片起点正常重基,源视频时间、硬件时间戳、动作含义保持一致。旧版本文件和记录通过任务级历史保留标记保护,后续送检和重切也不会由普通版本清理删除。 + +部署无数据库迁移,不自动修改已送检、已审核任务,也不代替客户送检。现有在审任务需要正常退回至标注后使用此入口;无需重新填写动作。更新现有在审结果并保持原审核状态,需要独立的、明确授权的数据修订流程,不属于本次部署。 + +验证覆盖:候选识别、非法选择拒绝且不占用任务、重叠候选、真实持久队列与媒体生成、导出 JSON 动作与时间戳保留,以及原版本所有文件逐字节未变。 diff --git a/docs/verification/video-presigned-playback-20260926.md b/docs/verification/video-presigned-playback-20260926.md new file mode 100644 index 00000000..021c3e3d --- /dev/null +++ b/docs/verification/video-presigned-playback-20260926.md @@ -0,0 +1,59 @@ +# Video playback: signed BOS delivery and recovery + +## Problem and final behavior + +Video Range requests were copied through the API's shared public egress and +two-minute response deadline. Focused single-camera inspection hid the other +video visually but continued loading and playing it. After a native media +failure, seeking cleared the message without reloading the failed element. + +- Authorized BOS video GETs now return a non-cacheable 307 to the existing + one-hour, read-only presigned URL, including valid Range requests. The browser + reads video bytes from BOS. Task/object authorization remains unchanged. +- HEAD, invalid ranges, explicit transcoding, structured previews, local files, + and backends without presigning retain their existing behavior. +- Focused single-camera inspection attaches only the selected source. Camera + changes align to the existing logical frame clock; clicking the already + selected camera does not interrupt playback. +- Media failures get at most two automatic reload attempts per unsuccessful + recovery. Seeking or the inline reload control also retries. Reloading uses the + original authorized route with a cache-busting parameter to obtain a fresh + signature, rather than reusing the redirected URL. External signatures are not + modified. Errors clear only once the requested frame is ready. +- Playback intent, the logical position and the mounted editor survive reloads. + No task records, slice boundaries, action judgments, source media or delivery + JSON are changed by this fix. + +## Verification + +The backend regression initially observed HTTP 206 instead of 307. The real +component browser regression initially found a src on the hidden right camera. +Both pass after the fix. + +- Backend object storage content, signed Range routing, anonymous authorization, + task evidence isolation, run artifact content and preview compatibility tests + pass. +- TypeScript check and production web build pass. +- Frontend suite: 612 tests, 599 passing, the same 13 pre-existing failures before + and after this change; failure names are recorded in the private release log. +- `tests/browser/video-playback-recovery.mjs` uses local synthetic video and the + real shared workbench in annotation, quality and acceptance viewing modes. It + checks hidden-stream absence, camera switching, repeated selection, transient + and persistent failures, bounded retries, frame positioning, explicit retry + and unsaved description preservation. +- Pre-release read-only browser checks confirmed actual BOS signed redirects + play successfully. All customer write requests were blocked before sending. + +## Release and rollback + +The user authorized this release and requested backups. Local private release +artifacts are under `local-data/verification/presigned-video-20260926/` in the +primary repository. Deployment, service and configuration snapshots retain +their pre-release versions. The API is updated by immutable image digest only, +with a compare-and-swap guard against overwriting a concurrent release. Other +deployments and customer data are outside this release. + +The saved inverse image patch is the rollback operation. Production verification +must remain read-only; report observed capacity separately from a full 12-user +business-workflow acceptance. No pull request or Git push is included in this +authorization. diff --git a/internal/api/annotation_blank_delivery_test.go b/internal/api/annotation_blank_delivery_test.go index db06ad36..2c726104 100644 --- a/internal/api/annotation_blank_delivery_test.go +++ b/internal/api/annotation_blank_delivery_test.go @@ -47,7 +47,7 @@ func TestContinuousBlankActionsReachReviewAndPublishedJSON(t *testing.T) { } } -func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation) { +func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation, consolidate ...bool) { t.Helper() t.Setenv("ANNOTATION_PROCESSOR_MODE", "external") f, root := annotationProcessorFixture(t) @@ -103,6 +103,11 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot } } slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: 4}} + var selected []string + if len(consolidate) > 0 && consolidate[0] { + slices = []domain.SliceSegment{{ID: "left", Name: "测试切片", Start: 0, End: 1}, {ID: "invalid", Name: "测试切片", Start: 1, End: 2}, {ID: "right", Name: "测试切片", Start: 2, End: 4}} + selected = []string{"invalid"} + } version := f.item.Version if reuse { response := f.request(http.MethodPost, endpoint+"/slice-confirmations", f.annotator, map[string]any{"version": version, "sliceSegments": slices}) @@ -111,14 +116,40 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot } version = waitFor(false) } + priorFiles := map[string]string{} + if reuse && len(selected) > 0 { + priorRoot := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, "revision-000001") + err := filepath.Walk(priorRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + data, e := os.ReadFile(path) + if e != nil { + return e + } + priorFiles[path] = string(data) + } + return nil + }) + if err != nil || len(priorFiles) == 0 { + t.Fatalf("prior evidence unavailable: %v", err) + } + } response := f.request(http.MethodPost, endpoint+"/combined-submissions", f.annotator, map[string]any{ "version": version, "requireSliceCompletion": true, - "sliceSegments": slices, "segments": segments, + "sliceSegments": slices, "segments": segments, "consolidateInvalidSliceIds": selected, }) if response.Code != http.StatusAccepted { t.Fatalf("blank submission: %d %s", response.Code, response.Body.String()) } waitFor(true) + for path, original := range priorFiles { + data, e := os.ReadFile(path) + if e != nil || string(data) != original { + t.Fatalf("old revision overwritten: %s %v", path, e) + } + } response = f.request(http.MethodGet, endpoint, f.reviewer, nil) if response.Code != http.StatusOK { t.Fatalf("quality read: %d %s", response.Code, response.Body.String()) @@ -149,7 +180,7 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot if reuse { expectedGeneration = 2 } - if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != reuse { + if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != (reuse && len(selected) == 0) { t.Fatalf("expected generation %d reuse %v: %s", expectedGeneration, reuse, response.Body.String()) } path := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, fmt.Sprintf("revision-%06d", generation), "slices", "slice-000001", "annotation.json") @@ -180,3 +211,9 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot } } } + +func TestInvalidSliceConsolidationRebuildsMediaAndPreservesPublishedActions(t *testing.T) { + actions := []domain.AnnotationSegment{{ID: "left-action", Start: 0, End: 1, Status: "described", Description: "拿起"}, {ID: "invalid-action", Start: 1, End: 2, Status: "invalid", IsInvalid: true, Description: "无效片段"}, {ID: "right-action", Start: 2, End: 4, Status: "described", Description: "放下"}} + want := []blankActionExpectation{{"left-action", "described", "拿起", false, 0, 1, 1000000, 2000000}, {"invalid-action", "invalid", "无效片段", true, 1, 2, 2000000, 3000000}, {"right-action", "described", "放下", false, 2, 4, 3000000, 5000000}} + checkBlankActionsDelivery(t, true, actions, want, true) +} diff --git a/internal/api/annotation_current.go b/internal/api/annotation_current.go index 157bee5e..94150540 100644 --- a/internal/api/annotation_current.go +++ b/internal/api/annotation_current.go @@ -23,6 +23,11 @@ func (s *server) retainCurrentAnnotationData(ctx context.Context, task domain.An if item.LegacyTaskID != task.ID || item.OrganizationID != task.OrganizationID || item.RunID != task.RunID { return fmt.Errorf("annotation retention scope mismatch") } + // A lossless consolidation pins its original evidence for future reslicing + // and submissions, not just the first post-consolidation cleanup. + if item.Subject.Payload["preserveAnnotationRevisionHistory"] == true { + return nil + } // Disposition history is review evidence. Keep its private prior generations; // this does not add selectable delivery versions or alter project deletion. if history, ok := item.Subject.Payload["sliceDispositionHistory"]; ok && history != nil { diff --git a/internal/api/annotation_invalid_slice_consolidation.go b/internal/api/annotation_invalid_slice_consolidation.go new file mode 100644 index 00000000..ae8ec142 --- /dev/null +++ b/internal/api/annotation_invalid_slice_consolidation.go @@ -0,0 +1,82 @@ +package api + +import ( + "fmt" + "sort" + "strings" + + "droboticflow/internal/domain" +) + +// Consolidation is an explicit submission choice, never a migration or a +// cancel-cut. Action records are not passed by pointer or rewritten here. +func consolidateInvalidActionSlices(input []domain.SliceSegment, actions []domain.AnnotationSegment, selected []string) ([]domain.SliceSegment, []domain.SliceOperation, error) { + if len(selected) == 0 { + return input, nil, nil + } + ordered := append([]domain.SliceSegment(nil), input...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].Start < ordered[j].Start }) + eligible := map[string]int{} + for i := 1; i < len(ordered)-1; i++ { + l, c, r := ordered[i-1], ordered[i], ordered[i+1] + if l.IsDiscarded() || c.IsDiscarded() || r.IsDiscarded() || strings.TrimSpace(c.Name) == "" || l.Name != c.Name || r.Name != c.Name || l.End != c.Start || c.End != r.Start { + continue + } + cursor, has, valid := c.Start, false, true + inner := append([]domain.AnnotationSegment(nil), actions...) + sort.Slice(inner, func(i, j int) bool { return inner[i].Start < inner[j].Start }) + for _, a := range inner { + if a.End <= c.Start || a.Start >= c.End { + continue + } + if a.Start != cursor || a.End > c.End || a.End <= a.Start || !a.IsInvalid { + valid = false + break + } + cursor, has = a.End, true + } + if valid && has && cursor == c.End { + eligible[c.ID] = i + } + } + edges := make([]bool, len(ordered)) + seen := map[string]bool{} + for _, id := range selected { + i, ok := eligible[id] + if !ok || seen[id] { + return nil, nil, fmt.Errorf("所选无效区间的相邻切片已变化,请重新核对后提交") + } + seen[id] = true + edges[i-1], edges[i] = true, true + } + var result []domain.SliceSegment + var operations []domain.SliceOperation + for i := 0; i < len(ordered); { + j := i + for j < len(ordered)-1 && edges[j] { + j++ + } + merged := ordered[i] + if j > i { + // Keep original IDs in audit provenance; reuse the leftmost stable ID. + merged.SourceIDs = nil + provenance := map[string]bool{} + ids := []string{} + for k := i; k <= j; k++ { + ids = append(ids, ordered[k].ID) + for _, id := range append([]string{ordered[k].ID}, ordered[k].SourceIDs...) { + if !provenance[id] { + merged.SourceIDs = append(merged.SourceIDs, id) + provenance[id] = true + } + } + } + merged.End = ordered[j].End + merged.EndTimestampUS = ordered[j].EndTimestampUS + operations = append(operations, domain.SliceOperation{Type: "consolidate_invalid_actions", SegmentIDs: ids, Before: append([]domain.SliceSegment(nil), ordered[i:j+1]...), After: []domain.SliceSegment{merged}}) + } + result = append(result, merged) + i = j + 1 + } + return result, operations, nil +} diff --git a/internal/api/annotation_invalid_slice_consolidation_test.go b/internal/api/annotation_invalid_slice_consolidation_test.go new file mode 100644 index 00000000..b668e7b8 --- /dev/null +++ b/internal/api/annotation_invalid_slice_consolidation_test.go @@ -0,0 +1,82 @@ +package api + +import ( + "encoding/json" + "reflect" + "testing" + + "droboticflow/internal/domain" +) + +func TestInvalidSliceConsolidationPreservesActionsAndAudit(t *testing.T) { + slices := []domain.SliceSegment{{ID: "a", Name: "收银", Start: 0, End: 2, StartTimestampUS: 100}, {ID: "b", Name: "收银", Start: 2, End: 3}, {ID: "c", Name: "收银", Start: 3, End: 5, EndTimestampUS: 900}, {ID: "d", Name: "其他", Start: 5, End: 9}} + actions := []domain.AnnotationSegment{{ID: "x", Start: 0, End: 2, Description: "拿起商品", Phrases: []string{"细节"}}, {ID: "y", Start: 2, End: 3, IsInvalid: true, Description: "无效片段", Status: "invalid", StartTimestampUS: 200, EndTimestampUS: 300}, {ID: "z", Start: 3, End: 5, Description: "放下商品"}} + before, _ := json.Marshal(actions) + original, _ := json.Marshal(slices) + got, ops, err := consolidateInvalidActionSlices(slices, actions, []string{"b"}) + if err != nil || len(got) != 2 || got[0].Start != 0 || got[0].End != 5 || got[0].Name != "收银" || got[0].StartTimestampUS != 100 || got[0].EndTimestampUS != 900 { + t.Fatalf("%+v %v", got, err) + } + after, _ := json.Marshal(actions) + if string(before) != string(after) { + t.Fatal("action JSON changed") + } + unchanged, _ := json.Marshal(slices) + if string(original) != string(unchanged) { + t.Fatal("input slices mutated") + } + if !reflect.DeepEqual(ops[0].Before, slices[:3]) || ops[0].Type != "consolidate_invalid_actions" { + t.Fatal("audit lost") + } + if !reflect.DeepEqual(got[1], slices[3]) { + t.Fatal("unrelated slice changed") + } + kept, ops, err := consolidateInvalidActionSlices(slices, actions, nil) + if err != nil || len(ops) != 0 || !reflect.DeepEqual(kept, slices) { + t.Fatal("implicit consolidation") + } + for name, change := range map[string]func([]domain.SliceSegment, []domain.AnnotationSegment){ + "discard": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[1].DeliveryDisposition = "discard" }, + "different-name": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Name = "盘点" }, + "gap": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Start = 3.1 }, + "valid-action": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].IsInvalid = false }, + "partial-invalid": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].End = 2.5 }, + } { + t.Run(name, func(t *testing.T) { + s := append([]domain.SliceSegment(nil), slices...) + a := append([]domain.AnnotationSegment(nil), actions...) + change(s, a) + if _, _, err := consolidateInvalidActionSlices(s, a, []string{"b"}); err == nil { + t.Fatal("unsafe consolidation accepted") + } + }) + } + if _, _, err = consolidateInvalidActionSlices(slices, actions, []string{"b", "b"}); err == nil { + t.Fatal("duplicate accepted") + } +} + +func TestInvalidSliceConsolidationOverlappingCandidates(t *testing.T) { + var ss []domain.SliceSegment + var aa []domain.AnnotationSegment + for i, id := range []string{"a", "b", "c", "d", "e"} { + ss = append(ss, domain.SliceSegment{ID: id, Name: "盘点", Start: float64(i), End: float64(i + 1)}) + aa = append(aa, domain.AnnotationSegment{ID: id, Start: float64(i), End: float64(i + 1), IsInvalid: i == 1 || i == 3}) + } + got, ops, err := consolidateInvalidActionSlices(ss, aa, []string{"b", "d"}) + if err != nil || len(got) != 1 || got[0].End != 5 || len(ops) != 1 || len(ops[0].Before) != 5 { + t.Fatalf("%+v %v", got, err) + } +} + +func TestInvalidSliceConsolidationRejectsStaleSelectionBeforeReservation(t *testing.T) { + f := continuousFixture(t) + r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": []domain.SliceSegment{{ID: "s", Name: "测试切片", Start: 0, End: 10}}, "segments": []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Description: "动作"}}, "consolidateInvalidSliceIds": []string{"missing"}}) + if r.Code != 400 { + t.Fatalf("%d %s", r.Code, r.Body.String()) + } + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + if item.Version != f.item.Version || item.PendingCombinedSubmissionID != "" { + t.Fatal("invalid request mutated task") + } +} diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index 3d507e9b..90ebf469 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -1055,6 +1055,7 @@ type sliceRevisionInput struct { // Slice correction and semantic labels are persisted together only after both // validate, so a quality reviewer never receives one without the other. type combinedAnnotationSubmissionInput struct { + ConsolidateInvalidSliceIDs []string `json:"consolidateInvalidSliceIds,omitempty"` // Set only by the internal cut job; never accepted from JSON. sliceOnly bool Version int `json:"version"` @@ -2056,6 +2057,22 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri renderValidationError(writer, issues...) return } + // Original slices, names, disposition and actions have all been validated. + // Only an explicit user selection may change the outer partition. Retain + // action IDs, descriptions, invalid flags and absolute timestamp boundaries. + if len(payload.ConsolidateInvalidSliceIDs) > 0 { + merged, operations, mergeErr := consolidateInvalidActionSlices(payload.SliceSegments, payload.Segments, payload.ConsolidateInvalidSliceIDs) + if mergeErr != nil { + renderValidationError(writer, validationIssue{Field: "consolidateInvalidSliceIds", Message: mergeErr.Error()}) + return + } + if issues := validateDescriptionIntervals(merged, payload.Segments, false); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + payload.SliceSegments = merged + payload.SliceOperations = append(payload.SliceOperations, operations...) + } payloadSHA, hashErr := fanyuV5CombinedPayloadSHA(payload) if hashErr != nil { renderError(writer, http.StatusInternalServerError, hashErr) @@ -2201,6 +2218,9 @@ func (server *server) completeCombinedAnnotationSubmission(writer http.ResponseW if humanItem.Subject.Payload == nil { humanItem.Subject.Payload = map[string]any{} } + if len(payload.ConsolidateInvalidSliceIDs) > 0 { + humanItem.Subject.Payload["preserveAnnotationRevisionHistory"] = true + } humanItem.Subject.Payload["materialization"] = materialization.Payload if workflow := mapValue(humanItem.Subject.Payload["sliceWorkflow"]); len(workflow) > 0 { workflow["materialization"] = materialization.Payload @@ -2999,7 +3019,19 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri return true } } - if operationFromRequest(request) != nil { server.planQualityReviewDecision(writer, request, *task, *item, batch, payload); return true } + if operationFromRequest(request) != nil { + server.planQualityReviewDecision(writer, request, *task, *item, batch, payload) + return true + } + returnAuthor := "" + if payload.Decision == domain.AnnotationReviewRejected { + var err error + returnAuthor, err = server.reworkAuthor(request.Context(), *item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return true + } + } now := nowISO() review := domain.AnnotationReviewRecord{ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: task.CurrentRevision, Stage: domain.AnnotationReviewStageQuality, Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, ProblemSliceIDs: payload.ProblemSliceIDs, CreatedAt: now} review.ReviewRound = item.ReviewRound @@ -3007,7 +3039,7 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri review.NextRevision = task.CurrentRevision + 1 review.RejectTargets = []string{"annotation_combined"} task.Status = domain.AnnotationTaskStatusRework - task.AssignedUserID = "" + task.AssignedUserID = returnAuthor } else { // Quality approval is not packaging approval. The root task is accepted // and waits for the one aggregate pre-package acceptance item. @@ -3039,6 +3071,10 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri _ = server.store.UpsertAnnotationTask(request.Context(), *task) } else { item.Status = domain.HumanItemRework + item.AssignedUserID = returnAuthor + if returnAuthor != "" { + item.ClaimedAt = now + } } } else { item.Status = domain.HumanItemAccepted @@ -3131,14 +3167,18 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r renderJSON(writer, http.StatusForbidden, map[string]string{"error": string(requiredRole) + " assignment is required"}) return } - if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) { return } + if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) { + return + } humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "") if !claimed { return } var prepared bool request, prepared = server.prepareReviewDecision(writer, request, *humanItem, "quality", payload.ReviewDecisionInput, payload, string(payload.Decision)) - if !prepared { return } + if !prepared { + return + } defer server.finishReviewDecisionPreparation(request) finishReviewWrite, writeAllowed := server.beginReviewWrite(writer, request, *humanItem) if !writeAllowed { diff --git a/internal/api/object_storage_browser.go b/internal/api/object_storage_browser.go index 4aa13082..fe3c77ca 100644 --- a/internal/api/object_storage_browser.go +++ b/internal/api/object_storage_browser.go @@ -2358,6 +2358,16 @@ func (server *server) handleObjectStorageContent(writer http.ResponseWriter, req renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr) return } + // Browser media requests normally include Range, including open-ended + // ranges. Redirect those too: proxying them shares the API's public + // bandwidth and fixed response deadline across all active viewers. + // Authorization and object resolution above still precede URL issuance. + if request.Method == http.MethodGet && summary.Kind == "video" && !previewOnly && redirectURL != "" { + writer.Header().Set("Cache-Control", "private, no-store") + writer.Header().Set("Referrer-Policy", "no-referrer") + http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) + return + } if request.Method == http.MethodHead || hasRange { if !hasRange && summary.SizeBytes > 0 { requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1} diff --git a/internal/api/project_deletion.go b/internal/api/project_deletion.go index 094e1d68..957d75b4 100644 --- a/internal/api/project_deletion.go +++ b/internal/api/project_deletion.go @@ -18,7 +18,9 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" + "sync" "time" ) @@ -452,9 +454,14 @@ func (s *server) deleteProjectTarget(ctx context.Context, plan domain.ProjectDel } return nil case "bos": - if err := s.validateDeletionReferences(ctx, plan); err != nil { - return err - } + // validateDeletionReferences is already re-run once by + // executeProjectDeletion immediately before it starts processing + // non-runtime targets (the runtimeDone gate) — that is the right + // point to catch a run created between plan-build and execution + // referencing this path. Re-running the whole org-wide validation + // again here, per bos target, made a project with a few thousand + // BOS targets take hours (each call re-fetches and re-scans every + // other run in the organization). browser, conn, cleanup, err := s.resolveObjectStorageBrowser(ctx, t.ConnectionID) if err != nil { return err @@ -641,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s } } } - runtimeDone := false - for index := range task.Plan.Targets { - target := &task.Plan.Targets[index] - if target.Status == "done" { - continue + targetConcurrency := 12 + if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" { + if n, e := strconv.Atoi(v); e == nil && n > 0 { + targetConcurrency = n } - if target.Kind != "runtime" && !runtimeDone { - deadline := time.Now().Add(150 * time.Second) - for { - items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID}) - if e != nil { - fail(e) - return - } - busy := false - for _, id := range task.Plan.RunIDs { - run, e := s.store.GetRun(ctx, id) - if e != nil { - fail(e) - return - } - if run != nil && run.ExecutionLeaseOwner != "" { - until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt) - if e != nil || until.After(time.Now()) { - busy = true - } - } - } - for i := range items { - item := &items[i] - if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil { - fail(e) - return - } - w := mapValue(item.Subject.Payload["sliceWorkflow"]) - if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" { - busy = true - } - } - if !busy { - break - } - if time.Now().After(deadline) { - fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试")) - return - } - select { - case <-ctx.Done(): - fail(ctx.Err()) - return - case <-time.After(2 * time.Second): - } + } + // processBatch runs deleteProjectTarget for the not-yet-done targets at + // indices with up to targetConcurrency in flight at once. Each target's + // Status/Error and every persist() call is guarded by mu, so concurrent + // workers never race on shared task state; deleteProjectTarget itself + // (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs + // unlocked. A single project's deletion used to process every target + // strictly one at a time, which made stopping thousands of real runs + // take as long as thousands of sequential K8s round trips. + var mu sync.Mutex + processBatch := func(indices []int) error { + sem := make(chan struct{}, targetConcurrency) + var wg sync.WaitGroup + var firstErr error + for _, index := range indices { + mu.Lock() + skip := task.Plan.Targets[index].Status == "done" || firstErr != nil + mu.Unlock() + if skip { + continue } - // Catch work admitted immediately before the project lock. - for _, step := range task.Plan.Targets { - if step.Kind == "runtime" { - if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil { - fail(e) - return - } + sem <- struct{}{} + wg.Add(1) + go func(index int) { + defer wg.Done() + defer func() { <-sem }() + mu.Lock() + if firstErr != nil { + mu.Unlock() + return } - } - // Scoped exactly as before (projectId match only, no workflowId - // fallback — see the loop below) but via an indexed query - // instead of an organization-wide scan. - fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil) + target := &task.Plan.Targets[index] + target.Error = "" + target.Status = "running" + persistErr := persist() + snapshot := *target + mu.Unlock() + if persistErr != nil { + mu.Lock() + if firstErr == nil { + firstErr = persistErr + } + mu.Unlock() + cancel() + return + } + runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot) + mu.Lock() + target = &task.Plan.Targets[index] + if runErr != nil { + target.Status = "failed" + target.Error = runErr.Error() + if firstErr == nil { + firstErr = runErr + } + } else { + target.Status = "done" + } + _ = persist() + mu.Unlock() + }(index) + } + wg.Wait() + return firstErr + } + indicesByKind := map[string][]int{} + nonRuntimePending := false + for index, t := range task.Plan.Targets { + indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index) + if t.Kind != "runtime" && t.Status != "done" { + nonRuntimePending = true + } + } + if e := processBatch(indicesByKind["runtime"]); e != nil { + fail(e) + return + } + // Re-run the phase-transition safety checks (wait for in-flight human + // work, re-verify runtime absence, revalidate references) exactly once + // per execution attempt, right before touching any file/storage target + // — the same point the original sequential loop ran them at. + if nonRuntimePending { + deadline := time.Now().Add(150 * time.Second) + for { + items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID}) if e != nil { fail(e) return } - known := map[string]bool{} + busy := false for _, id := range task.Plan.RunIDs { - known[id] = true + run, e := s.store.GetRun(ctx, id) + if e != nil { + fail(e) + return + } + if run != nil && run.ExecutionLeaseOwner != "" { + until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt) + if e != nil || until.After(time.Now()) { + busy = true + } + } } - for _, r := range fresh { - if r.ProjectID == task.Plan.ProjectID && !known[r.ID] { - fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID)) + for i := range items { + item := &items[i] + if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil { + fail(e) + return + } + w := mapValue(item.Subject.Payload["sliceWorkflow"]) + if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" { + busy = true + } + } + if !busy { + break + } + if time.Now().After(deadline) { + fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试")) + return + } + select { + case <-ctx.Done(): + fail(ctx.Err()) + return + case <-time.After(2 * time.Second): + } + } + // Catch work admitted immediately before the project lock. + for _, step := range task.Plan.Targets { + if step.Kind == "runtime" { + if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil { + fail(e) return } } - if e = s.validateDeletionReferences(ctx, task.Plan); e != nil { - fail(e) - return - } - runtimeDone = true } - target.Error = "" - target.Status = "running" - if e := persist(); e != nil { - cancel() - return - } - if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil { - target.Status = "failed" - target.Error = e.Error() + // Scoped exactly as before (projectId match only, no workflowId + // fallback — see the loop below) but via an indexed query + // instead of an organization-wide scan. + fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil) + if e != nil { fail(e) return } - target.Status = "done" - if e := persist(); e != nil { - cancel() + known := map[string]bool{} + for _, id := range task.Plan.RunIDs { + known[id] = true + } + for _, r := range fresh { + if r.ProjectID == task.Plan.ProjectID && !known[r.ID] { + fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID)) + return + } + } + if e = s.validateDeletionReferences(ctx, task.Plan); e != nil { + fail(e) + return + } + } + for _, kind := range []string{"workspace", "bos", "records"} { + if e := processBatch(indicesByKind[kind]); e != nil { + fail(e) return } } diff --git a/internal/api/project_deletion_test.go b/internal/api/project_deletion_test.go index 62db2d0b..5ad95e9a 100644 --- a/internal/api/project_deletion_test.go +++ b/internal/api/project_deletion_test.go @@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) { t.Fatal("raw data removed") } } + +// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the +// processBatch concurrency added to executeProjectDeletion: with many runs +// (many runtime + workspace targets processed by a worker pool instead of +// one at a time), the task must still end up "completed" with every target +// "done" and every run's workspace directory actually removed — i.e. the +// concurrent rewrite produces the same end state as the old sequential +// loop, just faster. +func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) { + ctx := context.Background() + db := store.NewMemoryStore() + p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"}) + if e != nil { + t.Fatal(e) + } + wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID}) + if e != nil { + t.Fatal(e) + } + s := &server{store: db, workspaceMountPath: t.TempDir()} + const runCount = 30 + dirs := make([]string, 0, runCount) + for i := 0; i < runCount; i++ { + runID := fmt.Sprintf("run-concurrent-fixture-%02d", i) + if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil { + t.Fatal(e) + } + dir, e := safeRunWorkspace(s.workspaceMountPath, runID) + if e != nil { + t.Fatal(e) + } + if e = os.MkdirAll(dir, 0700); e != nil { + t.Fatal(e) + } + dirs = append(dirs, dir) + } + plan, e := s.buildProjectDeletionPlan(ctx, p) + if e != nil { + t.Fatal(e) + } + if len(plan.Blockers) > 0 { + t.Fatal(plan.Blockers) + } + task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"}) + if e != nil { + t.Fatal(e) + } + old := projectDeletionRuntimeFactory + defer func() { projectDeletionRuntimeFactory = old }() + projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) { + return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil + } + claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute))) + if e != nil || !ok { + t.Fatal(e) + } + s.executeProjectDeletion(*claimed, "owner") + finished, _ := db.GetProjectDeletion(ctx, task.ID) + if finished.Status != "completed" { + t.Fatalf("%+v", finished) + } + for _, target := range finished.Plan.Targets { + if target.Status != "done" { + t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target) + } + } + for _, dir := range dirs { + if _, e = os.Stat(dir); !os.IsNotExist(e) { + t.Fatalf("workspace %s not removed", dir) + } + } + if existing, _ := db.GetProject(ctx, p.ID); existing != nil { + t.Fatal("project remains") + } +} diff --git a/internal/api/review_decision_plan.go b/internal/api/review_decision_plan.go index 60548ade..9b55c97a 100644 --- a/internal/api/review_decision_plan.go +++ b/internal/api/review_decision_plan.go @@ -8,6 +8,15 @@ import ( ) func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Request, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, batch *domain.HumanBatchRecord, payload annotationReviewInput) { + returnAuthor := "" + if payload.Decision == domain.AnnotationReviewRejected { + var err error + returnAuthor, err = s.reworkAuthor(r.Context(), item) + if err != nil { + renderError(w, http.StatusInternalServerError, err) + return + } + } op := operationFromRequest(r) now := op.CreatedAt beforeTask, beforeItem := task, item @@ -26,6 +35,10 @@ func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Reques item.ReviewRound++ item.CurrentStageID = "annotation_combined" item.Status = domain.HumanItemRework + item.AssignedUserID, task.AssignedUserID = returnAuthor, returnAuthor + if returnAuthor != "" { + item.ClaimedAt = now + } if item.ReviewRound > item.MaxReworkRounds { item.Status = domain.HumanItemEscalated task.Status = domain.AnnotationTaskStatusRejected diff --git a/internal/api/rework_assignment.go b/internal/api/rework_assignment.go new file mode 100644 index 00000000..b4aa7ba4 --- /dev/null +++ b/internal/api/rework_assignment.go @@ -0,0 +1,41 @@ +package api + +import ( + "context" + "droboticflow/internal/domain" + "strings" +) + +// Resolve before recording a decision: transient lookup failures must not silently +// turn a return-to-author decision into a public-pool assignment. +func (s *server) reworkAuthor(ctx context.Context, item domain.HumanItemRecord) (string, error) { + if item.LatestAuthorUserID == "" || item.ReviewRound+1 > item.MaxReworkRounds { + return "", nil + } + user, err := s.store.GetUser(ctx, item.LatestAuthorUserID) + if err != nil { + return "", err + } + if user == nil || user.OrganizationID != item.OrganizationID || user.Status != domain.UserStatusActive || user.Role != domain.UserRoleAnnotator || user.SupplierID == "" { + return "", nil + } + if item.SupplierID != "" && item.SupplierID != user.SupplierID { + return "", nil + } + supplier, err := s.store.GetSupplier(ctx, user.SupplierID) + if err != nil { + return "", err + } + if supplier == nil || supplier.OrganizationID != item.OrganizationID || !strings.EqualFold(supplier.Status, "active") { + return "", nil + } + members, err := s.annotationMembershipsForUser(ctx, item.OrganizationID, item.ProjectID, user.ID) + if err != nil { + return "", err + } + task := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, ProcessingScope: item.ProcessingScope} + if !membershipHasRole(members, task, domain.AnnotationRoleAnnotator) { + return "", nil + } + return user.ID, nil +} diff --git a/internal/api/rework_assignment_test.go b/internal/api/rework_assignment_test.go new file mode 100644 index 00000000..06b20c51 --- /dev/null +++ b/internal/api/rework_assignment_test.go @@ -0,0 +1,85 @@ +package api + +import ( + "droboticflow/internal/domain" + "encoding/json" + "net/http" + "testing" +) + +func TestQualityRejectionReturnsToEligibleAuthor(t *testing.T) { + for _, durable := range []bool{false, true} { + for _, scenario := range []string{"eligible", "disabled", "wrong-supplier", "no-membership", "supplier-inactive", "no-author", "escalated"} { + t.Run(scenario+map[bool]string{true: "-durable", false: "-legacy"}[durable], func(t *testing.T) { + f := decisionFixture(t) + f.item.LatestAuthorUserID = f.annotator.ID + f.item.SupplierID = f.annotator.SupplierID + f.task.SupplierID = f.annotator.SupplierID + f.item.MaxReworkRounds = 5 + switch scenario { + case "disabled": + f.annotator.Status = domain.UserStatusDisabled + f.store.UpsertUser(f.ctx, f.annotator) + case "wrong-supplier": + f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: "other", OrganizationID: f.item.OrganizationID, Name: "Other", Status: "active"}) + f.annotator.SupplierID = "other" + f.annotator.ID = "different-supplier-author" + f.item.LatestAuthorUserID = f.annotator.ID + f.store.UpsertUser(f.ctx, f.annotator) + case "no-membership": + f.store.DeleteProjectMember(f.ctx, "member-"+f.annotator.ID) + case "supplier-inactive": + f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: f.annotator.SupplierID, OrganizationID: f.item.OrganizationID, Name: "Inactive", Status: "inactive"}) + case "no-author": + f.item.LatestAuthorUserID = "" + case "escalated": + f.item.MaxReworkRounds = 1 + } + f.store.UpsertHumanItem(f.ctx, f.item) + f.store.UpsertAnnotationTask(f.ctx, f.task) + input := map[string]any{"stage": "quality", "decision": "rejected", "reasonCodes": []string{"manual_review_reject"}, "comment": "请修正", "problemSliceIds": []string{"left"}} + endpoint := "/api/plugins/annotation-qa/tasks/" + f.task.ID + "/reviews" + var code int + var body string + if durable { + version, headers := openReviewDraftSession(t, f) + input["operationId"] = "return-author" + input["draftVersion"] = 0 + input["resultVersion"] = version + r := reviewSessionRequest(f, http.MethodPost, endpoint, f.reviewer, input, headers) + code = r.Code + body = r.Body.String() + } else { + r := f.request(http.MethodPost, endpoint, f.reviewer, input) + code = r.Code + body = r.Body.String() + } + if code != 201 { + t.Fatalf("review %d %s", code, body) + } + item, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + task, _ := f.store.GetAnnotationTask(f.ctx, f.task.ID) + expected := "" + if scenario == "eligible" { + expected = f.annotator.ID + } + if item.AssignedUserID != expected || task.AssignedUserID != expected { + t.Fatalf("return owner item=%q task=%q want=%q", item.AssignedUserID, task.AssignedUserID, expected) + } + if item.CurrentRevision != f.item.CurrentRevision { + t.Fatal("annotation revision changed") + } + if expected != "" { + r := f.request(http.MethodGet, "/api/human-items?pool=mine&projectId="+item.ProjectID, f.annotator, nil) + var p struct { + Items []domain.HumanItemRecord `json:"items"` + } + json.Unmarshal(r.Body.Bytes(), &p) + if len(p.Items) != 1 { + t.Fatalf("returned task missing in author's pool: %s", r.Body.String()) + } + } + }) + } + } +} diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 3b853094..6fa3fd93 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -10449,7 +10449,7 @@ func TestObjectStorageContentSupportsRangeAndHeadWithoutReadingWholeObject(t *te if item == nil || item.ID != connection.ID { t.Fatalf("unexpected storage connection: %#v", item) } - return fakeBrowser, func() {}, nil + return &unsignedVideoBrowser{fakeBrowser}, func() {}, nil } defer func() { storageConnectionBrowserResolver = originalResolver }() diff --git a/internal/api/video_playback_redirect_test.go b/internal/api/video_playback_redirect_test.go new file mode 100644 index 00000000..c62bde70 --- /dev/null +++ b/internal/api/video_playback_redirect_test.go @@ -0,0 +1,86 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// A backend without presigning still supports the existing Range proxy. +type unsignedVideoBrowser struct{ *fakeObjectStorageBrowser } + +func (b *unsignedVideoBrowser) BuildPreviewURL(context.Context, string, string) (string, error) { + return "", nil +} + +func TestVideoPlaybackRedirectsRangesWithoutProxyingBytes(t *testing.T) { + ctx := context.Background() + data := store.NewMemoryStore() + connection, err := data.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ + Name: "video-test", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test", + Bucket: "videos", AccessKeyID: "test-ak", SecretAccessKey: "test-sk", + }) + if err != nil { + t.Fatal(err) + } + browser := newFakeObjectStorageBrowser() + key := "videos|capture.mp4" + browser.objects[key] = domain.ObjectStorageObjectSummary{ + Bucket: "videos", Key: "capture.mp4", Name: "capture.mp4", Kind: "video", + SizeBytes: 10, ContentType: "video/mp4", + } + browser.content[key] = []byte("0123456789") + old := storageConnectionBrowserResolver + defer func() { storageConnectionBrowserResolver = old }() + storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + return browser, nil, nil + } + handler := NewHandler(data, nil) + path := "/api/object-storage/content?connectionId=" + url.QueryEscape(connection.ID) + "&bucket=videos&key=capture.mp4" + for _, byteRange := range []string{"bytes=0-", "bytes=2-5", "bytes=-4", ""} { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("Range", byteRange) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusTemporaryRedirect { + t.Fatalf("range %q: status %d, want 307", byteRange, response.Code) + } + if response.Header().Get("Location") != "https://example.com/preview/videos/capture.mp4" { + t.Fatal("expected presigned storage target") + } + if response.Header().Get("Cache-Control") != "private, no-store" { + t.Fatal("signed redirects must not be cached") + } + } + if browser.readCalls[key] != 0 || browser.readRangeCalls[key] != 0 { + t.Fatal("API must not transfer any video bytes") + } + // Invalid ranges remain rejected locally; metadata-only reads remain HEAD. + for _, tc := range []struct { + method, byteRange string + status int + }{ + {http.MethodGet, "bytes=20-", 416}, {http.MethodHead, "", 200}, + } { + request := httptest.NewRequest(tc.method, path, nil) + request.Header.Set("Range", tc.byteRange) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != tc.status || response.Header().Get("Location") != "" { + t.Fatalf("method %s range %q: status %d", tc.method, tc.byteRange, response.Code) + } + } + // The original authorization gate is still in front of URL issuance. + t.Setenv("DROBOTICFLOW_REQUIRE_SESSION_AUTH", "true") + authenticatedHandler := NewHandler(data, nil) + response := httptest.NewRecorder() + authenticatedHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil)) + if response.Code != http.StatusUnauthorized || response.Header().Get("Location") != "" { + t.Fatalf("anonymous request leaked video URL: status %d", response.Code) + } +} diff --git a/internal/domain/types.go b/internal/domain/types.go index 563cddf3..acfdd0d9 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -872,6 +872,12 @@ type RunRecord struct { Error string `json:"error,omitempty" bson:"error,omitempty"` CreatedAt string `json:"createdAt" bson:"createdAt"` UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + // ArchivedAt marks a terminal run excluded from org-wide scans (e.g. the + // deletion-safety reference check) once it's old enough that its data no + // longer needs to participate in day-to-day queries. The record itself is + // untouched — archiving never deletes it; only deleting the run's own + // project does that. + ArchivedAt string `json:"archivedAt,omitempty" bson:"archivedAt,omitempty"` } type RunSummary struct { diff --git a/internal/store/account_admin_guard.go b/internal/store/account_admin_guard.go index fee6a107..d0e79270 100644 --- a/internal/store/account_admin_guard.go +++ b/internal/store/account_admin_guard.go @@ -84,7 +84,7 @@ func (store *MongoStore) updateAdministrator(ctx context.Context, item domain.Us _, e = store.users.UpdateOne(sc, immutableSupplierFilter(item.ID, item.SupplierID), bson.M{"$set": fields}) return nil, e }) - if mongo.IsDuplicateKeyError(err) { + if isPhoneUniqueDuplicateKeyError(err) { return ErrDuplicateUserPhone } return err diff --git a/internal/store/account_mutation_lock.go b/internal/store/account_mutation_lock.go index 4b812480..72f22993 100644 --- a/internal/store/account_mutation_lock.go +++ b/internal/store/account_mutation_lock.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "os" + "strconv" + "strings" "time" "droboticflow/internal/domain" @@ -14,22 +16,60 @@ import ( var ErrAccountMutationBusy = errors.New("account mutation is busy or requires recovery") -// A durable, per-resource mutex also works on standalone MongoDB. It deliberately -// has no expiring lease: a paused process must never resume after another writer -// has stolen its lock. After a process crash or ambiguous network result, an -// operator must verify that owner has stopped and inspect the affected resource -// before clearing that exact owner's lock. Normal exits release their own lock. +// accountMutationLockLeaseDuration bounds how long a held lock survives its +// owner going silent (crash, pod replacement, OOM) before another process +// may reclaim it. Renewed well before expiry by a background ticker while +// the guarded operation is still running (see withAccountMutationLock), so +// a live owner's lease never lapses out from under it — only a dead one's +// does. Configurable because "how long is safe to wait before auto- +// reclaiming" is an operational judgment call, not a constant. +func accountMutationLockLeaseDuration() time.Duration { + const fallback = 2 * time.Minute + raw := strings.TrimSpace(os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS")) + if raw == "" { + return fallback + } + seconds, err := strconv.Atoi(raw) + if err != nil || seconds <= 0 { + return fallback + } + return time.Duration(seconds) * time.Second +} + +// A durable, per-resource mutex also works on standalone MongoDB. It carries +// an expiring lease (accountMutationLockLeaseDuration) renewed periodically +// by a background ticker for as long as the guarded operation keeps running, +// so a live owner's lock is never reclaimed out from under it — only a lock +// whose owner stopped renewing (crashed, was replaced, or otherwise went +// silent) becomes reclaimable, once its lease lapses. Reclaiming a lapsed +// lease always assigns a fresh owner id, so the original holder's own +// release (guarded by "owner": owner) can never match afterward, and never +// touches the new holder's lock — the fresh id already acts as the fencing +// token a naive TTL-lock would be missing. A crash or ambiguous network +// result during the guarded write is still surfaced for the caller to +// verify, exactly as before; only the "must wait for an operator to notice +// and clear it by hand" half of the old design is gone. func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, operation func(context.Context) (any, error)) (result any, err error) { col := s.db.Collection("accountMutationLocks") _, e := col.InsertOne(ctx, bson.M{"_id": key, "locked": false}) if e != nil && !mongo.IsDuplicateKeyError(e) { return nil, e } + leaseDuration := accountMutationLockLeaseDuration() owner := domain.NewID("mutation") waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() for { - r, e := col.UpdateOne(waitCtx, bson.M{"_id": key, "locked": false}, bson.M{"$set": bson.M{"locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"), "startedAt": time.Now().UTC()}}) + now := time.Now().UTC() + filter := bson.M{"_id": key, "$or": bson.A{ + bson.M{"locked": false}, + bson.M{"locked": true, "leaseExpiresAt": bson.M{"$lt": now}}, + }} + update := bson.M{"$set": bson.M{ + "locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"), + "startedAt": now, "leaseExpiresAt": now.Add(leaseDuration), + }} + r, e := col.UpdateOne(waitCtx, filter, update) if e != nil { return nil, e } @@ -42,14 +82,52 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op case <-time.After(25 * time.Millisecond): } } + // Renew well inside the lease window (a fixed fraction of it) so normal + // scheduling jitter never lets a live owner's lease lapse; only an + // owner that stops renewing entirely — because it's gone — loses the + // lock. Mirrors the project-deletion task lease's own renewal ticker. + renewCtx, cancelRenew := context.WithCancel(context.Background()) + renewalDone := make(chan struct{}) + go func() { + defer close(renewalDone) + ticker := time.NewTicker(leaseDuration / 4) + defer ticker.Stop() + for { + select { + case <-renewCtx.Done(): + return + case <-ticker.C: + renewalCtx, renewalCancel := context.WithTimeout(context.Background(), 5*time.Second) + r, e := col.UpdateOne(renewalCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"leaseExpiresAt": time.Now().UTC().Add(leaseDuration)}}) + renewalCancel() + if e != nil { + // A transient error (network blip, timeout) — not proof + // the lock was actually lost. Keep the lease as-is and + // retry on the next tick; giving up here on a live owner + // would reintroduce the exact "two concurrent writers" + // race the lease is meant to prevent. + continue + } + if r.MatchedCount != 1 { + // The document no longer matches our owner: someone else + // genuinely reclaimed it after our lease lapsed. Stop + // renewing; the release below will correctly no-op or + // surface the conflict. + return + } + } + } + }() defer func() { + cancelRenew() + <-renewalDone // Do not unlock an uncertain write and admit a conflicting operation. if ctx.Err() != nil || mongo.IsNetworkError(err) || mongo.IsTimeout(err) { return } releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second) defer releaseCancel() - r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": ""}}) + r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": "", "leaseExpiresAt": ""}}) if e != nil || r.MatchedCount != 1 { err = fmt.Errorf("%w: operation finished but lock release needs verification", ErrAccountMutationBusy) } diff --git a/internal/store/account_mutation_lock_mongo_test.go b/internal/store/account_mutation_lock_mongo_test.go new file mode 100644 index 00000000..8c039372 --- /dev/null +++ b/internal/store/account_mutation_lock_mongo_test.go @@ -0,0 +1,145 @@ +package store + +import ( + "context" + "errors" + "os" + "sync/atomic" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// This test deliberately uses MongoDB, not MemoryStore: the lease/reclaim +// behavior is a persisted-document race the in-memory store can't +// reproduce. Found live on Fanyu: a lock whose owning process was replaced +// mid-hold (a rolling worker deploy, or a crash) stayed "locked" forever — +// the original design had no expiring lease by intent, trading availability +// for a hand-verified recovery. This locks in the replacement: an expiring, +// renewed lease that only a genuinely dead owner ever loses. +func TestAccountMutationLockLeaseReclaimAndRenewal(t *testing.T) { + uri := os.Getenv("MONGO_INDEX_TEST_URI") + if uri == "" { + t.Skip("MONGO_INDEX_TEST_URI required for real Mongo lock regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(ctx) + db := os.Getenv("MONGO_INDEX_TEST_DB") + if db == "" { + t.Fatal("MONGO_INDEX_TEST_DB required") + } + dbHandle := client.Database(db) + defer dbHandle.Collection("accountMutationLocks").Drop(context.Background()) + s := &MongoStore{db: dbHandle} + + t.Run("normal acquire and release round-trips and unblocks the next waiter", func(t *testing.T) { + key := "lease-test-normal" + _, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "first", nil }) + if err != nil { + t.Fatalf("first acquire: %v", err) + } + var doc bson.M + if err := dbHandle.Collection("accountMutationLocks").FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil { + t.Fatal(err) + } + if doc["locked"] != false { + t.Fatalf("expected lock released after normal completion, got %#v", doc) + } + if _, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "second", nil }); err != nil { + t.Fatalf("second acquire after release: %v", err) + } + }) + + t.Run("a lease that lapsed because its owner went silent is reclaimable", func(t *testing.T) { + key := "lease-test-lapsed" + col := dbHandle.Collection("accountMutationLocks") + // Simulate a crashed holder: locked, with a lease already in the past. + if _, err := col.InsertOne(ctx, bson.M{ + "_id": key, "locked": true, "owner": "mutation-dead-owner", "process": "dead-pod", + "startedAt": time.Now().UTC().Add(-time.Hour), "leaseExpiresAt": time.Now().UTC().Add(-time.Minute), + }); err != nil { + t.Fatal(err) + } + claimed := make(chan struct{}) + result, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { + close(claimed) + return "reclaimed", nil + }) + select { + case <-claimed: + default: + t.Fatal("operation never ran — lapsed lease was not reclaimed") + } + if err != nil { + t.Fatalf("reclaim after lapse: %v", err) + } + if result != "reclaimed" { + t.Fatalf("unexpected result: %v", result) + } + var doc bson.M + if err := col.FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil { + t.Fatal(err) + } + if doc["locked"] != false { + t.Fatalf("expected lock released after reclaim completed, got %#v", doc) + } + }) + + t.Run("a live owner's lease is renewed before it can lapse, blocking a concurrent reclaim attempt", func(t *testing.T) { + key := "lease-test-renewed" + originalLease := os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS") + os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", "1") // renew ticks every ~250ms + defer func() { + if originalLease == "" { + os.Unsetenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS") + } else { + os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", originalLease) + } + }() + + holderStarted := make(chan struct{}) + holderMayFinish := make(chan struct{}) + var reclaimAttempted atomic.Bool + var reclaimErr error + + done := make(chan struct{}) + go func() { + defer close(done) + _, e := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { + close(holderStarted) + <-holderMayFinish + return "held", nil + }) + if e != nil { + t.Errorf("long-held operation returned an error: %v", e) + } + }() + + <-holderStarted + // Outlive several renewal ticks (lease=1s, renew every ~250ms) while + // still holding the lock, then attempt a short-timeout reclaim: it + // must fail, proving the live holder's lease never lapsed. + time.Sleep(2500 * time.Millisecond) + reclaimCtx, reclaimCancel := context.WithTimeout(ctx, 500*time.Millisecond) + _, reclaimErr = s.withAccountMutationLock(reclaimCtx, key, func(context.Context) (any, error) { return "stolen", nil }) + reclaimCancel() + reclaimAttempted.Store(true) + close(holderMayFinish) + <-done + + if !reclaimAttempted.Load() { + t.Fatal("reclaim was never attempted") + } + if !errors.Is(reclaimErr, ErrAccountMutationBusy) { + t.Fatalf("expected the renewed lease to block a concurrent reclaim, got: %v", reclaimErr) + } + }) +} diff --git a/internal/store/account_phone_duplicate_error_mongo_test.go b/internal/store/account_phone_duplicate_error_mongo_test.go new file mode 100644 index 00000000..921c6565 --- /dev/null +++ b/internal/store/account_phone_duplicate_error_mongo_test.go @@ -0,0 +1,88 @@ +package store + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex reproduces the exact +// bug found live on Fanyu: a users collection that still carries a stale, +// pre-phone-migration email_unique index (this codebase no longer declares +// it, but an older database can still have it — see EnsureIndexes' +// dropObsoleteIndexes call). A duplicate-key error from that unrelated index +// must not be misreported as a phone conflict, or every account created +// after the first one missing an email field fails with a "phone must be +// unique" error that has nothing to do with the phone number entered. +func TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex(t *testing.T) { + uri := os.Getenv("MONGO_INDEX_TEST_URI") + if uri == "" { + t.Skip("MONGO_INDEX_TEST_URI required for real Mongo index regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri)) + if err != nil { + t.Fatal(err) + } + defer client.Disconnect(ctx) + db := os.Getenv("MONGO_INDEX_TEST_DB") + if db == "" { + t.Fatal("MONGO_INDEX_TEST_DB required") + } + c := client.Database(db).Collection(fmt.Sprintf("users_phone_dup_test_%d", time.Now().UnixNano())) + defer c.Drop(context.Background()) + + // A pre-migration-shaped users collection: phone_unique (current) plus a + // leftover, non-sparse email_unique (obsolete, but not yet dropped). + if _, err := c.Indexes().CreateMany(ctx, []mongo.IndexModel{ + uniqueIndex("phone_unique", bson.D{{Key: "phone", Value: 1}}), + uniqueIndex("email_unique", bson.D{{Key: "email", Value: 1}}), + }); err != nil { + t.Fatal(err) + } + + // First account has no email field — occupies email_unique's one null slot. + if _, err := c.InsertOne(ctx, bson.M{"id": "user-a", "phone": "13800000001"}); err != nil { + t.Fatal(err) + } + + // A second account, also missing email but with a genuinely distinct + // phone, collides only with the obsolete email index — must NOT be + // classified as a phone conflict. + _, err = c.InsertOne(ctx, bson.M{"id": "user-b", "phone": "13800000002"}) + if !mongo.IsDuplicateKeyError(err) { + t.Fatalf("expected a duplicate-key error from the stale email index, got: %v", err) + } + if isPhoneUniqueDuplicateKeyError(err) { + t.Fatalf("email_unique conflict misclassified as a phone conflict: %v", err) + } + + // A third account reusing user-a's actual phone must still be correctly + // classified as a real phone conflict. + _, err = c.InsertOne(ctx, bson.M{"id": "user-c", "phone": "13800000001", "email": "user-c@example.com"}) + if !mongo.IsDuplicateKeyError(err) { + t.Fatalf("expected a duplicate-key error from the phone index, got: %v", err) + } + if !isPhoneUniqueDuplicateKeyError(err) { + t.Fatalf("real phone_unique conflict not recognized: %v", err) + } + + if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil { + t.Fatal(err) + } + if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil { + t.Fatalf("dropping an already-absent index must be a no-op: %v", err) + } + + // With the obsolete index gone, a second account missing email succeeds. + if _, err := c.InsertOne(ctx, bson.M{"id": "user-d", "phone": "13800000003"}); err != nil { + t.Fatalf("account creation still blocked after dropping the obsolete index: %v", err) + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 0736cf78..6f884ad1 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -1281,23 +1281,34 @@ func (store *MemoryStore) ListRunArtifacts(ctx context.Context, status string) ( return items, nil } +// Excludes archived runs, matching MongoStore's ListRunArtifactsByOrganization +// (see ArchiveTerminalRuns) — this is the org-wide scan behind +// validateDeletionReferences' safety check. func (store *MemoryStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) { items, err := store.ListRunsByOrganization(ctx, organizationID, status) if err != nil { return nil, err } - hydrateRunArtifactsWhenNeeded(items) - return items, nil + kept := make([]domain.RunRecord, 0, len(items)) + for _, run := range items { + if run.ArchivedAt == "" { + kept = append(kept, run) + } + } + hydrateRunArtifactsWhenNeeded(kept) + return kept, nil } // ListRunArtifactsForProject mirrors the Mongo implementation's scope: runs // whose projectId matches projectID directly, or whose workflowId is in -// workflowIDs. +// workflowIDs. Deliberately includes archived runs — a project's own +// deletion must still clean up every run it ever had, archived or not. func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) { - all, err := store.ListRunArtifactsByOrganization(ctx, organizationID, "") + all, err := store.ListRunsByOrganization(ctx, organizationID, "") if err != nil { return nil, err } + hydrateRunArtifactsWhenNeeded(all) workflowSet := make(map[string]bool, len(workflowIDs)) for _, id := range workflowIDs { workflowSet[id] = true @@ -1311,6 +1322,29 @@ func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organi return items, nil } +// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns. +func (store *MemoryStore) ArchiveTerminalRuns(_ context.Context, olderThan time.Time) (int, error) { + store.mu.Lock() + defer store.mu.Unlock() + threshold := olderThan.UTC().Format(time.RFC3339Nano) + archived := 0 + for id, run := range store.runs { + if run.ArchivedAt != "" { + continue + } + if run.Status != domain.RunStatusCompleted && run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled { + continue + } + if run.EndedAt == "" || run.EndedAt >= threshold { + continue + } + run.ArchivedAt = nowISO() + store.runs[id] = run + archived++ + } + return archived, nil +} + func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunStatus]int, error) { store.mu.Lock() defer store.mu.Unlock() diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go index a749637a..3976005e 100644 --- a/internal/store/memory_test.go +++ b/internal/store/memory_test.go @@ -1211,3 +1211,102 @@ func TestListRunArtifactsForProjectScopesByProjectOrWorkflow(t *testing.T) { t.Fatalf("scope leaked unrelated runs: %#v", got) } } + +func TestArchiveTerminalRunsOnlyArchivesOldTerminalUnarchivedRuns(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano) + recent := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339Nano) + threshold := time.Now().Add(-30 * 24 * time.Hour) + + runs := []domain.RunRecord{ + {ID: "run-old-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old}, + {ID: "run-old-failed", OrganizationID: "org-1", Status: domain.RunStatusFailed, EndedAt: old}, + {ID: "run-old-cancelled", OrganizationID: "org-1", Status: domain.RunStatusCancelled, EndedAt: old}, + {ID: "run-recent-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: recent}, + {ID: "run-old-running", OrganizationID: "org-1", Status: domain.RunStatusRunning, EndedAt: ""}, + {ID: "run-already-archived", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old}, + } + for _, run := range runs { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + + archived, err := dataStore.ArchiveTerminalRuns(ctx, threshold) + if err != nil { + t.Fatalf("archive terminal runs: %v", err) + } + if archived != 3 { + t.Fatalf("expected 3 runs archived, got %d", archived) + } + + for _, id := range []string{"run-old-completed", "run-old-failed", "run-old-cancelled"} { + run, err := dataStore.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run %s: %v", id, err) + } + if run.ArchivedAt == "" { + t.Fatalf("expected %s to be archived", id) + } + } + for _, id := range []string{"run-recent-completed", "run-old-running"} { + run, err := dataStore.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run %s: %v", id, err) + } + if run.ArchivedAt != "" { + t.Fatalf("expected %s to remain unarchived, got %s", id, run.ArchivedAt) + } + } + + // Re-running must not re-archive (and must not double count) an already + // archived run or one that just got archived. + archivedAgain, err := dataStore.ArchiveTerminalRuns(ctx, threshold) + if err != nil { + t.Fatalf("archive terminal runs (second pass): %v", err) + } + if archivedAgain != 0 { + t.Fatalf("expected second pass to archive nothing, archived %d", archivedAgain) + } +} + +func TestListRunArtifactsByOrganizationExcludesArchivedRunsButForProjectStillIncludesThem(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + + old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano) + runs := []domain.RunRecord{ + {ID: "run-active", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old}, + {ID: "run-archived", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old}, + } + for _, run := range runs { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + + orgItems, err := dataStore.ListRunArtifactsByOrganization(ctx, "org-1", "") + if err != nil { + t.Fatalf("list run artifacts by organization: %v", err) + } + if len(orgItems) != 1 || orgItems[0].ID != "run-active" { + t.Fatalf("expected only the unarchived run from the org-wide scan, got %#v", orgItems) + } + + // A project's own deletion planning must still see its archived runs — + // archiving must never exempt a run from being cleaned up by its own + // project's deletion. + projectItems, err := dataStore.ListRunArtifactsForProject(ctx, "org-1", "project-a", nil) + if err != nil { + t.Fatalf("list run artifacts for project: %v", err) + } + got := map[string]bool{} + for _, item := range projectItems { + got[item.ID] = true + } + if !got["run-active"] || !got["run-archived"] { + t.Fatalf("expected project scope to include both runs regardless of archive state, got %#v", got) + } +} diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 076c7f4d..03179fcc 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -1073,8 +1073,12 @@ func (store *MongoStore) ListRunArtifactsForProject(ctx context.Context, organiz return items, nil } +// Excludes archived runs: this is the query behind validateDeletionReferences' +// org-wide safety scan, and an organization's run history is unbounded and +// never otherwise shrinks, so a run old enough to be archived is excluded +// here to keep that scan's cost from growing forever (see ArchiveTerminalRuns). func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) { - filter := bson.M{} + filter := bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}} if status != "" { filter["status"] = status } @@ -1098,6 +1102,20 @@ func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, org return items, nil } +// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns. +func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) { + filter := bson.M{ + "status": bson.M{"$in": bson.A{string(domain.RunStatusCompleted), string(domain.RunStatusFailed), string(domain.RunStatusCancelled)}}, + "endedAt": bson.M{"$ne": "", "$lt": olderThan.UTC().Format(time.RFC3339Nano)}, + "archivedAt": bson.M{"$in": bson.A{nil, ""}}, + } + result, err := store.runs.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"archivedAt": nowISO()}}) + if err != nil { + return 0, err + } + return int(result.ModifiedCount), nil +} + func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) { cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{ {{Key: "$group", Value: bson.M{ @@ -2237,12 +2255,21 @@ func (store *MongoStore) upsertUser(ctx context.Context, item domain.UserRecord, return ErrUserMutationConflict } } - if mongo.IsDuplicateKeyError(err) { + if isPhoneUniqueDuplicateKeyError(err) { return ErrDuplicateUserPhone } return err } +// isPhoneUniqueDuplicateKeyError reports whether err is a MongoDB duplicate-key +// error specifically from the users collection's phone_unique index, not just +// any unique-index conflict on that collection — a stale or unrelated index +// (e.g. a pre-migration email_unique index left over in an older database) +// would otherwise be misreported to the caller as "phone must be unique". +func isPhoneUniqueDuplicateKeyError(err error) bool { + return mongo.IsDuplicateKeyError(err) && strings.Contains(err.Error(), "phone_unique") +} + func (store *MongoStore) CreateSession(ctx context.Context, item domain.SessionRecord) (*domain.SessionRecord, error) { if strings.TrimSpace(item.ID) == "" { item.ID = domain.NewID("session") diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 72fcdcb4..3c64fcd4 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -118,6 +119,9 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { index("status_updatedAt", bson.D{{Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}), index("status_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}), index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}), + // Backs ArchiveTerminalRuns' periodic sweep for terminal, old, + // not-yet-archived runs. + index("status_endedAt", bson.D{{Key: "status", Value: 1}, {Key: "endedAt", Value: 1}}), index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}), index("workflowId_taskId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}), index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}), @@ -378,5 +382,27 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { return err } } + return dropObsoleteIndexes(ctx, store.users, "email_unique") +} + +// dropObsoleteIndexes removes indexes this codebase no longer declares. A +// database created before the accounts moved from email to phone as the +// sole identifier can still carry a non-sparse email_unique index (this +// function never creates it — only pre-migration deployments have it); left +// in place, the first account created without an email field occupies that +// index's one allowed null slot, and every account created after that +// mislabels as "phone must be unique" since the caller only checks whether +// *some* unique index on the collection conflicted. Missing-index errors +// are ignored since that's the desired end state. +func dropObsoleteIndexes(ctx context.Context, collection *mongo.Collection, names ...string) error { + for _, name := range names { + if _, err := collection.Indexes().DropOne(ctx, name); err != nil { + var cmdErr mongo.CommandError + if errors.As(err, &cmdErr) && cmdErr.Code == 27 { // IndexNotFound + continue + } + return err + } + } return nil } diff --git a/internal/store/store.go b/internal/store/store.go index a9516f13..1b419fb2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -104,6 +104,14 @@ type Store interface { // every run in the organization (see account-phone-login-era perf fix: // large orgs made the unscoped org-wide scan hang for minutes). ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) + // ArchiveTerminalRuns sets ArchivedAt on every not-yet-archived run whose + // status is completed/failed/cancelled and whose EndedAt is older than + // olderThan. Archiving never deletes the record — it only excludes it + // from the org-wide scans (see ListRunArtifactsByOrganization) that + // would otherwise keep growing as an organization's run history grows + // unbounded; a run's data is only actually removed when its project is + // deleted. Returns the number of runs archived. + ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error) ListRunSummaries(context.Context, string) ([]domain.RunSummary, error) ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error) diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 7e4202fd..49eafe7d 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -99,6 +99,9 @@ type Runtime struct { lastOrphanCheckNano atomic.Int64 pendingFailureAfter time.Duration missingMaxRecoveries int + runArchiveMinimumAge time.Duration + runArchiveInterval time.Duration + lastRunArchiveNano atomic.Int64 } type ProcessOnceStatus string @@ -203,6 +206,8 @@ func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName str orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute), pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute), missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3), + runArchiveMinimumAge: envDurationSeconds("WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS", 30*24*time.Hour), + runArchiveInterval: envDurationSeconds("WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS", time.Hour), }, nil } @@ -1076,6 +1081,39 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) { runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors) } +// archiveTerminalRuns periodically marks old, terminal runs as archived (see +// domain.RunRecord.ArchivedAt) so an organization's ever-growing run history +// doesn't keep inflating the cost of org-wide scans like +// validateDeletionReferences' safety check. Throttled the same way +// reconcileOrphanedRuntimes is, since this only needs to run occasionally, +// not on every poll tick. +func (runtime *Runtime) archiveTerminalRuns(ctx context.Context) { + now := time.Now().UTC() + interval := runtime.runArchiveInterval + if interval <= 0 { + interval = time.Hour + } + lastCheck := runtime.lastRunArchiveNano.Load() + if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval { + return + } + if !runtime.lastRunArchiveNano.CompareAndSwap(lastCheck, now.UnixNano()) { + return + } + minimumAge := runtime.runArchiveMinimumAge + if minimumAge <= 0 { + minimumAge = 30 * 24 * time.Hour + } + archived, err := runtime.store.ArchiveTerminalRuns(ctx, now.Add(-minimumAge)) + if err != nil { + fmt.Printf("[worker] archive terminal runs failed: %v\n", err) + return + } + if archived > 0 { + fmt.Printf("[worker] archived %d terminal run(s) older than %s\n", archived, minimumAge) + } +} + func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { startedAt := time.Now() var processErr error @@ -1138,6 +1176,7 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { } runtime.applyWorkflowRuntimePolicies(ctx, workflows) runtime.reconcileOrphanedRuntimes(ctx) + runtime.archiveTerminalRuns(ctx) queuedRuns := runRecordsFromSummaries(queuedPage.Items) selectedRunIDs := selectQueuedRunsForDispatchSnapshot( queuedRuns, diff --git a/k8s/annotation-processor.template.yaml b/k8s/annotation-processor.template.yaml index 76ff931d..c34493e2 100644 --- a/k8s/annotation-processor.template.yaml +++ b/k8s/annotation-processor.template.yaml @@ -6,6 +6,8 @@ metadata: namespace: __NAMESPACE__ spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: {app: droboticflow-annotation-processor} template: @@ -22,8 +24,8 @@ spec: command: [/usr/local/bin/droboticflow-api] env: - {name: ANNOTATION_PROCESSOR_MODE, value: processor} - - {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "2"} - - {name: ANNOTATION_INDEX_BACKFILL, value: "1"} + - {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "1"} + - {name: ANNOTATION_INDEX_BACKFILL, value: "0"} - {name: WORKSPACE_MOUNT_PATH, value: "__WORKSPACE_MOUNT_PATH__"} - {name: GLOBAL_BOS_ENDPOINT, value: "__BOS_ENDPOINT__"} - {name: ANNOTATION_WORKSPACE_CONNECTION_ID, value: "__ANNOTATION_WORKSPACE_CONNECTION_ID__"} diff --git a/scripts/deploy-envs/fanyu.env b/scripts/deploy-envs/fanyu.env index b13bf39a..a9ac384c 100644 --- a/scripts/deploy-envs/fanyu.env +++ b/scripts/deploy-envs/fanyu.env @@ -14,3 +14,6 @@ export WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}" export WORKSPACE_BOS_BUCKET="${WORKSPACE_BOS_BUCKET:-drobodata-fanyu}" export ANNOTATION_WORKSPACE_CONNECTION_ID="${ANNOTATION_WORKSPACE_CONNECTION_ID:-storage-d0a49fe4b9881c324f0640ab62359603}" export ANNOTATION_WORKSPACE_BUCKET="${ANNOTATION_WORKSPACE_BUCKET:-$WORKSPACE_BOS_BUCKET}" + +# Keep heavy slice materialization out of the web API. +export ANNOTATION_PROCESSOR_ENABLED="${ANNOTATION_PROCESSOR_ENABLED:-true}" diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index 1d49e9ed..84e45e3e 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -379,8 +379,6 @@ export function CaptureAnnotationWorkbench({ const [playbackRate, setPlaybackRate] = useState(1); const [showLeft, setShowLeft] = useState(initialCameraVisibility?.left ?? true); const [showRight, setShowRight] = useState(initialCameraVisibility?.right ?? Boolean(rightVideoSource)); - const effectiveShowLeft = focusedLayout || showLeft; - const effectiveShowRight = focusedLayout ? Boolean(rightVideoSource) : showRight; const cameraStorageKey = cameraPreferenceKey ? `${isQualityReview || providedReviewCaption ? "review" : "annotation"}-camera-default:${cameraPreferenceKey}` : ""; @@ -394,6 +392,7 @@ export function CaptureAnnotationWorkbench({ }; const [defaultCamera, setDefaultCamera] = useState(readCameraDefault); const initialInspection = () => { + if (!rightVideoSource) return "left" as const; if (providedReviewCaption && initialCameraVisibility) return initialCameraVisibility.left && initialCameraVisibility.right ? null @@ -404,7 +403,11 @@ export function CaptureAnnotationWorkbench({ return preference === "both" ? null : (preference as "left" | "right"); }; const [inspectionCamera, setInspectionCamera] = useState<"left" | "right" | null>(initialInspection); + const effectiveShowLeft = focusedLayout ? inspectionCamera !== "right" || !rightVideoSource : showLeft; + const effectiveShowRight = Boolean(rightVideoSource) && (focusedLayout ? inspectionCamera !== "left" : showRight); const inspectCamera = (camera: "left" | "right" | null) => { + if (camera === inspectionCamera) return; + transport.prepareCameraChange(); setInspectionCamera(camera); if (providedReviewCaption) onCameraVisibilityChange?.({ left: camera !== "right", right: camera !== "left" }); }; @@ -2650,7 +2653,7 @@ export function CaptureAnnotationWorkbench({ )} diff --git a/src/hooks/useCaptureVideoTransport.ts b/src/hooks/useCaptureVideoTransport.ts index 6f41c326..573e5c6a 100644 --- a/src/hooks/useCaptureVideoTransport.ts +++ b/src/hooks/useCaptureVideoTransport.ts @@ -34,6 +34,27 @@ export function useCaptureVideoTransport(options: Options) { const [isPositioning, setIsPositioning] = useState(false); const [mediaError, setMediaError] = useState(""); const targets = useRef<{ video: HTMLVideoElement; time: number; revision: number }[]>([]); + const retryTimer = useRef | null>(null); + const retryAttempts = useRef({ left: 0, right: 0 }); + const reloadSequence = useRef(0); + + function cancelRetry() { + if (retryTimer.current !== null) clearTimeout(retryTimer.current); + retryTimer.current = null; + } + function reloadTrack(video: HTMLVideoElement) { + const c = config.current; + const source = video === c.leftRef.current ? c.leftSource : c.rightSource; + if (!source) return; + const url = new URL(source, window.location.href); + // Return to the original authenticated route, never to currentSrc (which + // can be an expired BOS redirect). Do not alter external URL signatures. + if (url.origin === window.location.origin) { + url.searchParams.set("_mediaRetry", `${Date.now()}-${++reloadSequence.current}`); + } + video.src = url.href; + video.load(); + } function visibleTracks() { const c = config.current; @@ -66,6 +87,10 @@ export function useCaptureVideoTransport(options: Options) { } function seekTrack(target: (typeof targets.current)[number]) { const { video, time } = target; + if (video.error) { + reloadTrack(video); + return; + } if (video.readyState < 1) return; // loadedmetadata will finish this request. const next = physicalTarget(video, time); if (Math.abs(video.currentTime - next) > 0.000001) video.currentTime = next; @@ -77,19 +102,24 @@ export function useCaptureVideoTransport(options: Options) { ({ video, time }) => // HTMLMediaElement can quantize to microseconds. This tolerance is // only for readiness, never for the saved/visible annotation time. - video.seeking || video.readyState < 2 || Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002, + video.error || + video.seeking || + video.readyState < 2 || + Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002, ) ) return; const revision = targets.current[0].revision; targets.current = []; + setMediaError(""); + retryAttempts.current = { left: 0, right: 0 }; queue.complete(revision); } const [queue] = useState(() => createMediaSeekQueue({ dispatch(time: number, revision: number) { + cancelRetry(); setIsPositioning(true); - setMediaError(""); pauseElements(); targets.current = visibleTracks().map(({ video, toMedia }) => ({ video, time: toMedia(time), revision })); targets.current.forEach(seekTrack); @@ -105,6 +135,7 @@ export function useCaptureVideoTransport(options: Options) { ); useEffect( () => () => { + cancelRetry(); playing.current = false; queue.cancel(); targets.current = []; @@ -165,11 +196,25 @@ export function useCaptureVideoTransport(options: Options) { function play() { playing.current = true; setIsPlaying(true); + if (visibleTracks().some(({ video }) => video.error)) { + retry(); + return; + } if (!queue.isPending() && !scrubbing.current) playElements(); } + function retry() { + cancelRetry(); + retryAttempts.current = { left: 0, right: 0 }; + queue.cancel(); + queue.commit(position.current); + } + // Sources/visibility, not playhead updates, own resource lifetime. useEffect(() => { + cancelRetry(); + retryAttempts.current = { left: 0, right: 0 }; + setMediaError(""); queue.cancel(); targets.current = []; scrubbing.current = false; @@ -191,6 +236,7 @@ export function useCaptureVideoTransport(options: Options) { } queue.commit(position.current); return () => { + cancelRetry(); queue.cancel(); targets.current = []; }; @@ -210,12 +256,28 @@ export function useCaptureVideoTransport(options: Options) { onLoadedData: settle, onCanPlay: settle, onError(event: { currentTarget: HTMLVideoElement }) { - if (!event.currentTarget.getAttribute("src")) return; + if ( + !event.currentTarget.getAttribute("src") || + !visibleTracks().some(({ video }) => video === event.currentTarget) + ) + return; + cancelRetry(); queue.cancel(); targets.current = []; setIsPositioning(false); - setMediaError("视频加载失败,请重新定位或重新打开任务。"); - pause(); + setMediaError("视频加载失败,当前编辑已保留。"); + // Keep play intent and the logical position, so recovery does not + // remount the editor or jump to the beginning of the video. + pauseElements(); + if (config.current.enabled && retryAttempts.current[camera] < 2) { + const attempt = ++retryAttempts.current[camera]; + retryTimer.current = setTimeout(() => { + retryTimer.current = null; + queue.commit(position.current); + }, attempt * 750); + } else { + pause(); + } }, onPlay(event: { currentTarget: HTMLVideoElement }) { if (event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current) @@ -224,7 +286,13 @@ export function useCaptureVideoTransport(options: Options) { setIsPlaying(true); }, onPause(event: { currentTarget: HTMLVideoElement }) { - if (!event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current) + if ( + !event.currentTarget.paused || + !isMaster(event.currentTarget) || + queue.isPending() || + scrubbing.current || + retryTimer.current !== null + ) return; pause(); }, @@ -257,6 +325,7 @@ export function useCaptureVideoTransport(options: Options) { pause, pauseAtFrame, play, + retry, isPlaying, isPositioning, mediaError, diff --git a/src/lib/invalid-slice-consolidation.js b/src/lib/invalid-slice-consolidation.js new file mode 100644 index 00000000..0651dab3 --- /dev/null +++ b/src/lib/invalid-slice-consolidation.js @@ -0,0 +1,44 @@ +// Candidate discovery only. The server revalidates explicit selections and +// creates a new revision; viewing or marking an action invalid never merges. +export function invalidSliceConsolidationCandidates(slices, actions) { + const ordered = [...slices].sort((a, b) => a.start - b.start); + return ordered.flatMap((current, index) => { + const left = ordered[index - 1]; + const right = ordered[index + 1]; + if ( + !left || + !right || + [left, current, right].some((s) => s.deliveryDisposition === "discard") || + !current.name?.trim() || + left.name !== current.name || + right.name !== current.name || + left.end !== current.start || + current.end !== right.start + ) + return []; + const inner = actions + .filter((a) => a.end > current.start && a.start < current.end) + .sort((a, b) => a.start - b.start); + let cursor = current.start; + if ( + !inner.length || + !inner.every((a) => { + if (!a.isInvalid || a.start !== cursor || a.end <= a.start || a.end > current.end) return false; + cursor = a.end; + return true; + }) || + cursor !== current.end + ) + return []; + return [ + { + id: current.id, + name: current.name, + start: left.start, + end: right.end, + invalidStart: current.start, + invalidEnd: current.end, + }, + ]; + }); +} diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 531044ab..9ab9fa99 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -1,3 +1,4 @@ +import { invalidSliceConsolidationCandidates } from "../lib/invalid-slice-consolidation.js"; import { ReviewDecisionDialog } from "../components/ReviewDecisionDialog"; import { useReviewContinuation } from "../hooks/useReviewContinuation"; import { useReviewEntryIntent } from "../hooks/useReviewEntryIntent"; @@ -3157,12 +3158,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?: setError("请保存所有动作描述或明确设为无效"); return; } + const candidates = invalidSliceConsolidationCandidates(sliceSegments, segments); + const consolidateInvalidSliceIds: string[] = []; + for (const candidate of candidates) { + if ( + window.confirm( + `“${candidate.name}”中 ${candidate.invalidStart.toFixed(3)}–${candidate.invalidEnd.toFixed(3)} 秒的无效动作单独占了一个切片。\n\n是否与前后同名切片合并为 ${candidate.start.toFixed(3)}–${candidate.end.toFixed(3)} 秒的完整切片?全部动作描述、无效标记及时间保持不变。\n确定:合并外层;取消:保留原划分。`, + ) + ) + consolidateInvalidSliceIds.push(candidate.id); + } + if ( + candidates.length && + !window.confirm( + consolidateInvalidSliceIds.length + ? `已选择 ${consolidateInvalidSliceIds.length} 处无损合并。提交后重新生成审核视频,原版本保留。确认提交质检?` + : "保留原切片划分,确认提交质检?", + ) + ) + return; const currentTaskId = detail.task.id; const token = beginAnnotationMutation(currentTaskId); if (!token) return; clearEditHistory(); try { const submission = await annotationOperationsApi.submitCombined(detail.task.id, { + consolidateInvalidSliceIds, version: detailRef.current?.humanItem?.version ?? detail.humanItem.version, sliceSegments: applySliceStatuses( sliceSegments.map((slice) => ({ ...slice, taskLabelId: "", taskLabel: "" })), diff --git a/src/services/api/annotation-operations.ts b/src/services/api/annotation-operations.ts index 2d41d767..8f6f20a3 100644 --- a/src/services/api/annotation-operations.ts +++ b/src/services/api/annotation-operations.ts @@ -178,6 +178,7 @@ export const annotationOperationsApi = { taskId: string, payload: { version: number; + consolidateInvalidSliceIds?: string[]; sliceSegments: SliceSegment[]; sliceOperations?: SliceOperation[]; segments: AnnotationSegment[]; diff --git a/tests/browser/video-playback-recovery.mjs b/tests/browser/video-playback-recovery.mjs new file mode 100644 index 00000000..ee305420 --- /dev/null +++ b/tests/browser/video-playback-recovery.mjs @@ -0,0 +1,170 @@ +// Local real-component/media regression. Synthetic video only; no API writes. +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { chromium } from "playwright"; +const base = process.env.VIDEO_TEST_URL || "http://127.0.0.1:5196"; +assert(["127.0.0.1", "localhost"].includes(new URL(base).hostname), "local fixtures only"); +const out = process.env.VIDEO_TEST_OUTPUT || "/tmp/presigned-video-tests"; +await fs.mkdir(out, { recursive: true }); +const file = out + "/sample.mp4"; +try { + await fs.access(file); +} catch { + execFileSync("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=240x320:rate=30:duration=12", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + file, + ]); +} +const bytes = await fs.readFile(file); +const browser = await chromium.launch({ + executablePath: + process.env.PLAYWRIGHT_EXECUTABLE_PATH || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + headless: true, + args: ["--no-proxy-server"], +}); +try { + for (const mode of ["annotation", "quality", "acceptance"]) { + const page = await browser.newPage({ viewport: { width: 1413, height: 900 } }); + page.setDefaultTimeout(15000); + let fail = false, + failedRequests = 0; + const requests = [], + errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + await page.route("**/api/**", async (route) => { + const req = route.request(), + u = new URL(req.url()); + assert.equal(req.method(), "GET"); + if (!u.pathname.endsWith(".mp4")) return route.fulfill({ json: {} }); + requests.push({ path: u.pathname, retry: u.searchParams.get("_mediaRetry") }); + if (fail && u.pathname.includes("left")) { + failedRequests++; + return route.abort("failed"); + } + const match = /bytes=(\d+)-(\d*)/.exec(req.headers().range || ""); + const start = match ? Number(match[1]) : 0, + end = match?.[2] ? Math.min(Number(match[2]), bytes.length - 1) : bytes.length - 1; + return route.fulfill({ + status: match ? 206 : 200, + contentType: "video/mp4", + headers: { + "Accept-Ranges": "bytes", + "Content-Length": String(end - start + 1), + ...(match ? { "Content-Range": `bytes ${start}-${end}/${bytes.length}` } : {}), + }, + body: bytes.subarray(start, end + 1), + }); + }); + await page.route("**/__video_recovery", (route) => + route.fulfill({ + contentType: "text/html", + body: `
`, + }), + ); + await page.goto(base + "/__video_recovery", { waitUntil: "domcontentloaded" }); + const bench = page.getByTestId("capture-annotation-workbench"), + left = page.locator('video[data-camera="left"]'), + right = page.locator('video[data-camera="right"]'); + const settle = () => + page.waitForFunction(() => { + const videos = [...document.querySelectorAll("video")].filter((v) => v.getAttribute("src")); + return videos.length && videos.every((v) => v.readyState >= 2 && !v.seeking && !v.error); + }); + await bench.waitFor(); + await settle(); + assert.equal(await right.getAttribute("src"), null, "single-camera view must detach the hidden stream"); + assert.equal( + requests.filter((r) => r.path.includes("right")).length, + 0, + "hidden camera must make no media request", + ); + await bench.getByRole("button", { name: "播放", exact: true }).click(); + await page.waitForFunction(() => document.querySelector('video[data-camera="left"]').currentTime > 1); + const beforeSameCamera = await left.evaluate((v) => v.currentTime); + await bench.getByRole("button", { name: "放大左相机", exact: true }).click(); + await page.waitForFunction( + (t) => document.querySelector('video[data-camera="left"]').currentTime > t + 0.3, + beforeSameCamera, + ); + // One transient read failure recovers automatically and retains play intent. + fail = true; + await left.evaluate((v) => v.load()); + await page.waitForFunction(() => !!document.querySelector('video[data-camera="left"]').error); + fail = false; + await settle(); + await page.waitForFunction(() => !document.querySelector('video[data-camera="left"]').paused); + failedRequests = 0; + await bench.getByRole("button", { name: "放大右相机", exact: true }).click(); + await settle(); + assert.equal(await left.getAttribute("src"), null); + assert((await right.evaluate((v) => v.currentTime)) > 0.8, "camera switch preserves logical playback position"); + await bench.getByRole("button", { name: "恢复双画面", exact: true }).click(); + await settle(); + await bench.getByRole("button", { name: "暂停", exact: true }).click(); + await bench.getByRole("button", { name: "放大左相机", exact: true }).click(); + await settle(); + const held = await left.evaluate((v) => v.currentTime); + let input; + if (mode === "annotation") { + await bench.getByTestId("annotation-description-block").first().click(); + input = bench.getByRole("textbox", { name: "动作描述", exact: true }); + await input.fill("Unsubmitted draft must survive video recovery"); + } + const beforeFailure = await left.evaluate((v) => v.currentTime); + fail = true; + await left.evaluate((v) => v.load()); + await page.waitForTimeout(4500); + assert(failedRequests <= 3, `automatic retries must be bounded: ${failedRequests}`); + assert(await left.evaluate((v) => !!v.error), "persistent failure remains visible in the media element"); + assert(await bench.getByRole("alert").count(), "do not clear the failure before media recovers"); + fail = false; + await bench.getByRole("button", { name: "下一帧", exact: true }).click(); + await settle(); + assert( + requests.some((r) => r.retry), + "retry returns to original authorized URL for a fresh signature", + ); + assert( + Math.abs((await left.evaluate((v) => v.currentTime)) - beforeFailure) < 0.15, + "recovery preserves the frame clock", + ); + assert.equal(await left.evaluate((v) => v.paused), true, "paused users are not auto-played"); + if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery"); + // Explicit retry also works after the bounded automatic attempts are exhausted. + fail = true; + failedRequests = 0; + await left.evaluate((v) => v.load()); + await page.waitForTimeout(4500); + fail = false; + await bench.getByRole("button", { name: "重新加载视频", exact: true }).click(); + await settle(); + assert.equal(await bench.getByRole("alert").count(), 0); + if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery"); + assert.deepEqual(errors, []); + console.log(JSON.stringify({ mode, passed: true, held, requests: requests.length })); + await page.close(); + } +} finally { + await browser.close(); +} diff --git a/tests/frontend/capture-annotation-workbench.test.mjs b/tests/frontend/capture-annotation-workbench.test.mjs index 8cb0fdd5..ae5d0acd 100644 --- a/tests/frontend/capture-annotation-workbench.test.mjs +++ b/tests/frontend/capture-annotation-workbench.test.mjs @@ -67,8 +67,8 @@ test("both playhead sliders use latest-only pointer previews and exact final com }); test("hidden sources detach and right-only playback maps back to the canonical frame clock", () => { - assert.match(workbench, /const effectiveShowLeft = focusedLayout \|\| showLeft/); - assert.match(workbench, /const effectiveShowRight = focusedLayout \? Boolean\(rightVideoSource\) : showRight/); + // Real focused-camera resource lifetime is exercised by + // tests/browser/video-playback-recovery.mjs (hidden camera issues no request). assert.match(workbench, /src=\{effectiveShowLeft \? source : undefined\}/); assert.match(workbench, /src=\{effectiveShowRight \? rightSource : undefined\}/); assert.match(workbench, /mapCameraBoundaryTime\(rightMediaBoundaries, frameBoundaries, time\)/); diff --git a/tests/frontend/invalid-slice-consolidation.test.mjs b/tests/frontend/invalid-slice-consolidation.test.mjs new file mode 100644 index 00000000..ed1a6db0 --- /dev/null +++ b/tests/frontend/invalid-slice-consolidation.test.mjs @@ -0,0 +1,40 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { invalidSliceConsolidationCandidates as candidates } from "../../src/lib/invalid-slice-consolidation.js"; +const slices = ["a", "b", "c"].map((id, i) => ({ id, name: "盘点", start: i, end: i + 1 })); +const actions = [{ id: "invalid", start: 1, end: 2, isInvalid: true, description: "无效片段" }]; +test("identify invalid-only outer partition without modifying slices or action JSON", () => { + const before = JSON.stringify({ slices, actions }); + assert.deepEqual(candidates(slices, actions), [ + { id: "b", name: "盘点", start: 0, end: 3, invalidStart: 1, invalidEnd: 2 }, + ]); + assert.equal(JSON.stringify({ slices, actions }), before); +}); +test("never propose discarded, different-name, gapped or partially invalid partitions", () => { + assert.deepEqual( + candidates( + slices.map((s) => (s.id === "b" ? { ...s, deliveryDisposition: "discard" } : s)), + actions, + ), + [], + ); + assert.deepEqual( + candidates( + slices.map((s) => (s.id === "c" ? { ...s, name: "清洁" } : s)), + actions, + ), + [], + ); + assert.deepEqual( + candidates( + slices.map((s) => (s.id === "c" ? { ...s, start: 2.1 } : s)), + actions, + ), + [], + ); + assert.deepEqual(candidates(slices, [{ ...actions[0], end: 1.5 }]), []); + assert.deepEqual(candidates(slices, [{ ...actions[0], isInvalid: false }]), []); +}); +test("invalid inside a single business slice stays inside it", () => { + assert.deepEqual(candidates([{ id: "a", name: "盘点", start: 0, end: 3 }], actions), []); +});