🔀 Sync production video playback fixes into local workspace
# Conflicts: # scripts/deploy-envs/fanyu.env
This commit is contained in:
commit
c5cdd6c1da
13
docs/annotation-invalid-slice-consolidation.md
Normal file
13
docs/annotation-invalid-slice-consolidation.md
Normal file
@ -0,0 +1,13 @@
|
||||
# 无效动作与外层切片的无损整理
|
||||
|
||||
无效动作属于外层业务切片内部,设置无效不新增外层边界。旧版取消切分仍按既有规则清空合并区间判断;它不是无损整理入口。
|
||||
|
||||
送检前识别以下候选:连续三个保留切片名称完全相同、首尾精确相接,中间切片由明确无效的动作完整覆盖。用户逐项选择是否合并,最后再次确认提交。取消选择可保留原划分,不对所有同名切片做自动合并。舍弃、名称不同、缺口、部分无效都不匹配。
|
||||
|
||||
服务器重新验证选择,仅在已有领取权、版本、字典、覆盖与动作合法性校验全部通过后,改变送检版本的外层划分。动作数组不因合并改变:ID、描述、无效判断、起止边界、硬件时间戳均保留。重叠候选合成同一连续区间。操作历史记录原切片快照。
|
||||
|
||||
后台生成新的媒体和 JSON;不能把旧短片直接冒充合并后视频。JSON 中相对于切片起点的时间随新的切片起点正常重基,源视频时间、硬件时间戳、动作含义保持一致。旧版本文件和记录通过任务级历史保留标记保护,后续送检和重切也不会由普通版本清理删除。
|
||||
|
||||
部署无数据库迁移,不自动修改已送检、已审核任务,也不代替客户送检。现有在审任务需要正常退回至标注后使用此入口;无需重新填写动作。更新现有在审结果并保持原审核状态,需要独立的、明确授权的数据修订流程,不属于本次部署。
|
||||
|
||||
验证覆盖:候选识别、非法选择拒绝且不占用任务、重叠候选、真实持久队列与媒体生成、导出 JSON 动作与时间戳保留,以及原版本所有文件逐字节未变。
|
||||
59
docs/verification/video-presigned-playback-20260926.md
Normal file
59
docs/verification/video-presigned-playback-20260926.md
Normal file
@ -0,0 +1,59 @@
|
||||
# Video playback: signed BOS delivery and recovery
|
||||
|
||||
## Problem and final behavior
|
||||
|
||||
Video Range requests were copied through the API's shared public egress and
|
||||
two-minute response deadline. Focused single-camera inspection hid the other
|
||||
video visually but continued loading and playing it. After a native media
|
||||
failure, seeking cleared the message without reloading the failed element.
|
||||
|
||||
- Authorized BOS video GETs now return a non-cacheable 307 to the existing
|
||||
one-hour, read-only presigned URL, including valid Range requests. The browser
|
||||
reads video bytes from BOS. Task/object authorization remains unchanged.
|
||||
- HEAD, invalid ranges, explicit transcoding, structured previews, local files,
|
||||
and backends without presigning retain their existing behavior.
|
||||
- Focused single-camera inspection attaches only the selected source. Camera
|
||||
changes align to the existing logical frame clock; clicking the already
|
||||
selected camera does not interrupt playback.
|
||||
- Media failures get at most two automatic reload attempts per unsuccessful
|
||||
recovery. Seeking or the inline reload control also retries. Reloading uses the
|
||||
original authorized route with a cache-busting parameter to obtain a fresh
|
||||
signature, rather than reusing the redirected URL. External signatures are not
|
||||
modified. Errors clear only once the requested frame is ready.
|
||||
- Playback intent, the logical position and the mounted editor survive reloads.
|
||||
No task records, slice boundaries, action judgments, source media or delivery
|
||||
JSON are changed by this fix.
|
||||
|
||||
## Verification
|
||||
|
||||
The backend regression initially observed HTTP 206 instead of 307. The real
|
||||
component browser regression initially found a src on the hidden right camera.
|
||||
Both pass after the fix.
|
||||
|
||||
- Backend object storage content, signed Range routing, anonymous authorization,
|
||||
task evidence isolation, run artifact content and preview compatibility tests
|
||||
pass.
|
||||
- TypeScript check and production web build pass.
|
||||
- Frontend suite: 612 tests, 599 passing, the same 13 pre-existing failures before
|
||||
and after this change; failure names are recorded in the private release log.
|
||||
- `tests/browser/video-playback-recovery.mjs` uses local synthetic video and the
|
||||
real shared workbench in annotation, quality and acceptance viewing modes. It
|
||||
checks hidden-stream absence, camera switching, repeated selection, transient
|
||||
and persistent failures, bounded retries, frame positioning, explicit retry
|
||||
and unsaved description preservation.
|
||||
- Pre-release read-only browser checks confirmed actual BOS signed redirects
|
||||
play successfully. All customer write requests were blocked before sending.
|
||||
|
||||
## Release and rollback
|
||||
|
||||
The user authorized this release and requested backups. Local private release
|
||||
artifacts are under `local-data/verification/presigned-video-20260926/` in the
|
||||
primary repository. Deployment, service and configuration snapshots retain
|
||||
their pre-release versions. The API is updated by immutable image digest only,
|
||||
with a compare-and-swap guard against overwriting a concurrent release. Other
|
||||
deployments and customer data are outside this release.
|
||||
|
||||
The saved inverse image patch is the rollback operation. Production verification
|
||||
must remain read-only; report observed capacity separately from a full 12-user
|
||||
business-workflow acceptance. No pull request or Git push is included in this
|
||||
authorization.
|
||||
@ -47,7 +47,7 @@ func TestContinuousBlankActionsReachReviewAndPublishedJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation) {
|
||||
func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.AnnotationSegment, want []blankActionExpectation, consolidate ...bool) {
|
||||
t.Helper()
|
||||
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
@ -103,6 +103,11 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
|
||||
}
|
||||
}
|
||||
slices := []domain.SliceSegment{{ID: "whole", Name: "测试切片", Start: 0, End: 4}}
|
||||
var selected []string
|
||||
if len(consolidate) > 0 && consolidate[0] {
|
||||
slices = []domain.SliceSegment{{ID: "left", Name: "测试切片", Start: 0, End: 1}, {ID: "invalid", Name: "测试切片", Start: 1, End: 2}, {ID: "right", Name: "测试切片", Start: 2, End: 4}}
|
||||
selected = []string{"invalid"}
|
||||
}
|
||||
version := f.item.Version
|
||||
if reuse {
|
||||
response := f.request(http.MethodPost, endpoint+"/slice-confirmations", f.annotator, map[string]any{"version": version, "sliceSegments": slices})
|
||||
@ -111,14 +116,40 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
|
||||
}
|
||||
version = waitFor(false)
|
||||
}
|
||||
priorFiles := map[string]string{}
|
||||
if reuse && len(selected) > 0 {
|
||||
priorRoot := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, "revision-000001")
|
||||
err := filepath.Walk(priorRoot, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
data, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
priorFiles[path] = string(data)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil || len(priorFiles) == 0 {
|
||||
t.Fatalf("prior evidence unavailable: %v", err)
|
||||
}
|
||||
}
|
||||
response := f.request(http.MethodPost, endpoint+"/combined-submissions", f.annotator, map[string]any{
|
||||
"version": version, "requireSliceCompletion": true,
|
||||
"sliceSegments": slices, "segments": segments,
|
||||
"sliceSegments": slices, "segments": segments, "consolidateInvalidSliceIds": selected,
|
||||
})
|
||||
if response.Code != http.StatusAccepted {
|
||||
t.Fatalf("blank submission: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
waitFor(true)
|
||||
for path, original := range priorFiles {
|
||||
data, e := os.ReadFile(path)
|
||||
if e != nil || string(data) != original {
|
||||
t.Fatalf("old revision overwritten: %s %v", path, e)
|
||||
}
|
||||
}
|
||||
response = f.request(http.MethodGet, endpoint, f.reviewer, nil)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("quality read: %d %s", response.Code, response.Body.String())
|
||||
@ -149,7 +180,7 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
|
||||
if reuse {
|
||||
expectedGeneration = 2
|
||||
}
|
||||
if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != reuse {
|
||||
if generation != expectedGeneration || detail.HumanItem.Subject.Payload.Materialization.MetadataOnly != (reuse && len(selected) == 0) {
|
||||
t.Fatalf("expected generation %d reuse %v: %s", expectedGeneration, reuse, response.Body.String())
|
||||
}
|
||||
path := filepath.Join(root, "system", "runs", f.task.RunID, "human-materializations", f.item.ID, f.task.ID, fmt.Sprintf("revision-%06d", generation), "slices", "slice-000001", "annotation.json")
|
||||
@ -180,3 +211,9 @@ func checkBlankActionsDelivery(t *testing.T, reuse bool, segments []domain.Annot
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidSliceConsolidationRebuildsMediaAndPreservesPublishedActions(t *testing.T) {
|
||||
actions := []domain.AnnotationSegment{{ID: "left-action", Start: 0, End: 1, Status: "described", Description: "拿起"}, {ID: "invalid-action", Start: 1, End: 2, Status: "invalid", IsInvalid: true, Description: "无效片段"}, {ID: "right-action", Start: 2, End: 4, Status: "described", Description: "放下"}}
|
||||
want := []blankActionExpectation{{"left-action", "described", "拿起", false, 0, 1, 1000000, 2000000}, {"invalid-action", "invalid", "无效片段", true, 1, 2, 2000000, 3000000}, {"right-action", "described", "放下", false, 2, 4, 3000000, 5000000}}
|
||||
checkBlankActionsDelivery(t, true, actions, want, true)
|
||||
}
|
||||
|
||||
@ -23,6 +23,11 @@ func (s *server) retainCurrentAnnotationData(ctx context.Context, task domain.An
|
||||
if item.LegacyTaskID != task.ID || item.OrganizationID != task.OrganizationID || item.RunID != task.RunID {
|
||||
return fmt.Errorf("annotation retention scope mismatch")
|
||||
}
|
||||
// A lossless consolidation pins its original evidence for future reslicing
|
||||
// and submissions, not just the first post-consolidation cleanup.
|
||||
if item.Subject.Payload["preserveAnnotationRevisionHistory"] == true {
|
||||
return nil
|
||||
}
|
||||
// Disposition history is review evidence. Keep its private prior generations;
|
||||
// this does not add selectable delivery versions or alter project deletion.
|
||||
if history, ok := item.Subject.Payload["sliceDispositionHistory"]; ok && history != nil {
|
||||
|
||||
82
internal/api/annotation_invalid_slice_consolidation.go
Normal file
82
internal/api/annotation_invalid_slice_consolidation.go
Normal file
@ -0,0 +1,82 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
// Consolidation is an explicit submission choice, never a migration or a
|
||||
// cancel-cut. Action records are not passed by pointer or rewritten here.
|
||||
func consolidateInvalidActionSlices(input []domain.SliceSegment, actions []domain.AnnotationSegment, selected []string) ([]domain.SliceSegment, []domain.SliceOperation, error) {
|
||||
if len(selected) == 0 {
|
||||
return input, nil, nil
|
||||
}
|
||||
ordered := append([]domain.SliceSegment(nil), input...)
|
||||
sort.Slice(ordered, func(i, j int) bool { return ordered[i].Start < ordered[j].Start })
|
||||
eligible := map[string]int{}
|
||||
for i := 1; i < len(ordered)-1; i++ {
|
||||
l, c, r := ordered[i-1], ordered[i], ordered[i+1]
|
||||
if l.IsDiscarded() || c.IsDiscarded() || r.IsDiscarded() || strings.TrimSpace(c.Name) == "" || l.Name != c.Name || r.Name != c.Name || l.End != c.Start || c.End != r.Start {
|
||||
continue
|
||||
}
|
||||
cursor, has, valid := c.Start, false, true
|
||||
inner := append([]domain.AnnotationSegment(nil), actions...)
|
||||
sort.Slice(inner, func(i, j int) bool { return inner[i].Start < inner[j].Start })
|
||||
for _, a := range inner {
|
||||
if a.End <= c.Start || a.Start >= c.End {
|
||||
continue
|
||||
}
|
||||
if a.Start != cursor || a.End > c.End || a.End <= a.Start || !a.IsInvalid {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
cursor, has = a.End, true
|
||||
}
|
||||
if valid && has && cursor == c.End {
|
||||
eligible[c.ID] = i
|
||||
}
|
||||
}
|
||||
edges := make([]bool, len(ordered))
|
||||
seen := map[string]bool{}
|
||||
for _, id := range selected {
|
||||
i, ok := eligible[id]
|
||||
if !ok || seen[id] {
|
||||
return nil, nil, fmt.Errorf("所选无效区间的相邻切片已变化,请重新核对后提交")
|
||||
}
|
||||
seen[id] = true
|
||||
edges[i-1], edges[i] = true, true
|
||||
}
|
||||
var result []domain.SliceSegment
|
||||
var operations []domain.SliceOperation
|
||||
for i := 0; i < len(ordered); {
|
||||
j := i
|
||||
for j < len(ordered)-1 && edges[j] {
|
||||
j++
|
||||
}
|
||||
merged := ordered[i]
|
||||
if j > i {
|
||||
// Keep original IDs in audit provenance; reuse the leftmost stable ID.
|
||||
merged.SourceIDs = nil
|
||||
provenance := map[string]bool{}
|
||||
ids := []string{}
|
||||
for k := i; k <= j; k++ {
|
||||
ids = append(ids, ordered[k].ID)
|
||||
for _, id := range append([]string{ordered[k].ID}, ordered[k].SourceIDs...) {
|
||||
if !provenance[id] {
|
||||
merged.SourceIDs = append(merged.SourceIDs, id)
|
||||
provenance[id] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
merged.End = ordered[j].End
|
||||
merged.EndTimestampUS = ordered[j].EndTimestampUS
|
||||
operations = append(operations, domain.SliceOperation{Type: "consolidate_invalid_actions", SegmentIDs: ids, Before: append([]domain.SliceSegment(nil), ordered[i:j+1]...), After: []domain.SliceSegment{merged}})
|
||||
}
|
||||
result = append(result, merged)
|
||||
i = j + 1
|
||||
}
|
||||
return result, operations, nil
|
||||
}
|
||||
82
internal/api/annotation_invalid_slice_consolidation_test.go
Normal file
82
internal/api/annotation_invalid_slice_consolidation_test.go
Normal file
@ -0,0 +1,82 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func TestInvalidSliceConsolidationPreservesActionsAndAudit(t *testing.T) {
|
||||
slices := []domain.SliceSegment{{ID: "a", Name: "收银", Start: 0, End: 2, StartTimestampUS: 100}, {ID: "b", Name: "收银", Start: 2, End: 3}, {ID: "c", Name: "收银", Start: 3, End: 5, EndTimestampUS: 900}, {ID: "d", Name: "其他", Start: 5, End: 9}}
|
||||
actions := []domain.AnnotationSegment{{ID: "x", Start: 0, End: 2, Description: "拿起商品", Phrases: []string{"细节"}}, {ID: "y", Start: 2, End: 3, IsInvalid: true, Description: "无效片段", Status: "invalid", StartTimestampUS: 200, EndTimestampUS: 300}, {ID: "z", Start: 3, End: 5, Description: "放下商品"}}
|
||||
before, _ := json.Marshal(actions)
|
||||
original, _ := json.Marshal(slices)
|
||||
got, ops, err := consolidateInvalidActionSlices(slices, actions, []string{"b"})
|
||||
if err != nil || len(got) != 2 || got[0].Start != 0 || got[0].End != 5 || got[0].Name != "收银" || got[0].StartTimestampUS != 100 || got[0].EndTimestampUS != 900 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
after, _ := json.Marshal(actions)
|
||||
if string(before) != string(after) {
|
||||
t.Fatal("action JSON changed")
|
||||
}
|
||||
unchanged, _ := json.Marshal(slices)
|
||||
if string(original) != string(unchanged) {
|
||||
t.Fatal("input slices mutated")
|
||||
}
|
||||
if !reflect.DeepEqual(ops[0].Before, slices[:3]) || ops[0].Type != "consolidate_invalid_actions" {
|
||||
t.Fatal("audit lost")
|
||||
}
|
||||
if !reflect.DeepEqual(got[1], slices[3]) {
|
||||
t.Fatal("unrelated slice changed")
|
||||
}
|
||||
kept, ops, err := consolidateInvalidActionSlices(slices, actions, nil)
|
||||
if err != nil || len(ops) != 0 || !reflect.DeepEqual(kept, slices) {
|
||||
t.Fatal("implicit consolidation")
|
||||
}
|
||||
for name, change := range map[string]func([]domain.SliceSegment, []domain.AnnotationSegment){
|
||||
"discard": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[1].DeliveryDisposition = "discard" },
|
||||
"different-name": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Name = "盘点" },
|
||||
"gap": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { s[2].Start = 3.1 },
|
||||
"valid-action": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].IsInvalid = false },
|
||||
"partial-invalid": func(s []domain.SliceSegment, a []domain.AnnotationSegment) { a[1].End = 2.5 },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s := append([]domain.SliceSegment(nil), slices...)
|
||||
a := append([]domain.AnnotationSegment(nil), actions...)
|
||||
change(s, a)
|
||||
if _, _, err := consolidateInvalidActionSlices(s, a, []string{"b"}); err == nil {
|
||||
t.Fatal("unsafe consolidation accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, _, err = consolidateInvalidActionSlices(slices, actions, []string{"b", "b"}); err == nil {
|
||||
t.Fatal("duplicate accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidSliceConsolidationOverlappingCandidates(t *testing.T) {
|
||||
var ss []domain.SliceSegment
|
||||
var aa []domain.AnnotationSegment
|
||||
for i, id := range []string{"a", "b", "c", "d", "e"} {
|
||||
ss = append(ss, domain.SliceSegment{ID: id, Name: "盘点", Start: float64(i), End: float64(i + 1)})
|
||||
aa = append(aa, domain.AnnotationSegment{ID: id, Start: float64(i), End: float64(i + 1), IsInvalid: i == 1 || i == 3})
|
||||
}
|
||||
got, ops, err := consolidateInvalidActionSlices(ss, aa, []string{"b", "d"})
|
||||
if err != nil || len(got) != 1 || got[0].End != 5 || len(ops) != 1 || len(ops[0].Before) != 5 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidSliceConsolidationRejectsStaleSelectionBeforeReservation(t *testing.T) {
|
||||
f := continuousFixture(t)
|
||||
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/combined-submissions", f.annotator, map[string]any{"version": f.item.Version, "sliceSegments": []domain.SliceSegment{{ID: "s", Name: "测试切片", Start: 0, End: 10}}, "segments": []domain.AnnotationSegment{{ID: "a", Start: 0, End: 10, Description: "动作"}}, "consolidateInvalidSliceIds": []string{"missing"}})
|
||||
if r.Code != 400 {
|
||||
t.Fatalf("%d %s", r.Code, r.Body.String())
|
||||
}
|
||||
item, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if item.Version != f.item.Version || item.PendingCombinedSubmissionID != "" {
|
||||
t.Fatal("invalid request mutated task")
|
||||
}
|
||||
}
|
||||
@ -1055,6 +1055,7 @@ type sliceRevisionInput struct {
|
||||
// Slice correction and semantic labels are persisted together only after both
|
||||
// validate, so a quality reviewer never receives one without the other.
|
||||
type combinedAnnotationSubmissionInput struct {
|
||||
ConsolidateInvalidSliceIDs []string `json:"consolidateInvalidSliceIds,omitempty"`
|
||||
// Set only by the internal cut job; never accepted from JSON.
|
||||
sliceOnly bool
|
||||
Version int `json:"version"`
|
||||
@ -2056,6 +2057,22 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
// Original slices, names, disposition and actions have all been validated.
|
||||
// Only an explicit user selection may change the outer partition. Retain
|
||||
// action IDs, descriptions, invalid flags and absolute timestamp boundaries.
|
||||
if len(payload.ConsolidateInvalidSliceIDs) > 0 {
|
||||
merged, operations, mergeErr := consolidateInvalidActionSlices(payload.SliceSegments, payload.Segments, payload.ConsolidateInvalidSliceIDs)
|
||||
if mergeErr != nil {
|
||||
renderValidationError(writer, validationIssue{Field: "consolidateInvalidSliceIds", Message: mergeErr.Error()})
|
||||
return
|
||||
}
|
||||
if issues := validateDescriptionIntervals(merged, payload.Segments, false); len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
payload.SliceSegments = merged
|
||||
payload.SliceOperations = append(payload.SliceOperations, operations...)
|
||||
}
|
||||
payloadSHA, hashErr := fanyuV5CombinedPayloadSHA(payload)
|
||||
if hashErr != nil {
|
||||
renderError(writer, http.StatusInternalServerError, hashErr)
|
||||
@ -2201,6 +2218,9 @@ func (server *server) completeCombinedAnnotationSubmission(writer http.ResponseW
|
||||
if humanItem.Subject.Payload == nil {
|
||||
humanItem.Subject.Payload = map[string]any{}
|
||||
}
|
||||
if len(payload.ConsolidateInvalidSliceIDs) > 0 {
|
||||
humanItem.Subject.Payload["preserveAnnotationRevisionHistory"] = true
|
||||
}
|
||||
humanItem.Subject.Payload["materialization"] = materialization.Payload
|
||||
if workflow := mapValue(humanItem.Subject.Payload["sliceWorkflow"]); len(workflow) > 0 {
|
||||
workflow["materialization"] = materialization.Payload
|
||||
@ -2999,7 +3019,19 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri
|
||||
return true
|
||||
}
|
||||
}
|
||||
if operationFromRequest(request) != nil { server.planQualityReviewDecision(writer, request, *task, *item, batch, payload); return true }
|
||||
if operationFromRequest(request) != nil {
|
||||
server.planQualityReviewDecision(writer, request, *task, *item, batch, payload)
|
||||
return true
|
||||
}
|
||||
returnAuthor := ""
|
||||
if payload.Decision == domain.AnnotationReviewRejected {
|
||||
var err error
|
||||
returnAuthor, err = server.reworkAuthor(request.Context(), *item)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return true
|
||||
}
|
||||
}
|
||||
now := nowISO()
|
||||
review := domain.AnnotationReviewRecord{ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: task.CurrentRevision, Stage: domain.AnnotationReviewStageQuality, Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, ProblemSliceIDs: payload.ProblemSliceIDs, CreatedAt: now}
|
||||
review.ReviewRound = item.ReviewRound
|
||||
@ -3007,7 +3039,7 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri
|
||||
review.NextRevision = task.CurrentRevision + 1
|
||||
review.RejectTargets = []string{"annotation_combined"}
|
||||
task.Status = domain.AnnotationTaskStatusRework
|
||||
task.AssignedUserID = ""
|
||||
task.AssignedUserID = returnAuthor
|
||||
} else {
|
||||
// Quality approval is not packaging approval. The root task is accepted
|
||||
// and waits for the one aggregate pre-package acceptance item.
|
||||
@ -3039,6 +3071,10 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri
|
||||
_ = server.store.UpsertAnnotationTask(request.Context(), *task)
|
||||
} else {
|
||||
item.Status = domain.HumanItemRework
|
||||
item.AssignedUserID = returnAuthor
|
||||
if returnAuthor != "" {
|
||||
item.ClaimedAt = now
|
||||
}
|
||||
}
|
||||
} else {
|
||||
item.Status = domain.HumanItemAccepted
|
||||
@ -3131,14 +3167,18 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r
|
||||
renderJSON(writer, http.StatusForbidden, map[string]string{"error": string(requiredRole) + " assignment is required"})
|
||||
return
|
||||
}
|
||||
if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) { return }
|
||||
if server.replayReviewDecision(writer, request, task.ID, payload.ReviewDecisionInput, payload) {
|
||||
return
|
||||
}
|
||||
humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "")
|
||||
if !claimed {
|
||||
return
|
||||
}
|
||||
var prepared bool
|
||||
request, prepared = server.prepareReviewDecision(writer, request, *humanItem, "quality", payload.ReviewDecisionInput, payload, string(payload.Decision))
|
||||
if !prepared { return }
|
||||
if !prepared {
|
||||
return
|
||||
}
|
||||
defer server.finishReviewDecisionPreparation(request)
|
||||
finishReviewWrite, writeAllowed := server.beginReviewWrite(writer, request, *humanItem)
|
||||
if !writeAllowed {
|
||||
|
||||
@ -2358,6 +2358,16 @@ func (server *server) handleObjectStorageContent(writer http.ResponseWriter, req
|
||||
renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr)
|
||||
return
|
||||
}
|
||||
// Browser media requests normally include Range, including open-ended
|
||||
// ranges. Redirect those too: proxying them shares the API's public
|
||||
// bandwidth and fixed response deadline across all active viewers.
|
||||
// Authorization and object resolution above still precede URL issuance.
|
||||
if request.Method == http.MethodGet && summary.Kind == "video" && !previewOnly && redirectURL != "" {
|
||||
writer.Header().Set("Cache-Control", "private, no-store")
|
||||
writer.Header().Set("Referrer-Policy", "no-referrer")
|
||||
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
if request.Method == http.MethodHead || hasRange {
|
||||
if !hasRange && summary.SizeBytes > 0 {
|
||||
requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1}
|
||||
|
||||
@ -18,7 +18,9 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -452,9 +454,14 @@ func (s *server) deleteProjectTarget(ctx context.Context, plan domain.ProjectDel
|
||||
}
|
||||
return nil
|
||||
case "bos":
|
||||
if err := s.validateDeletionReferences(ctx, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
// validateDeletionReferences is already re-run once by
|
||||
// executeProjectDeletion immediately before it starts processing
|
||||
// non-runtime targets (the runtimeDone gate) — that is the right
|
||||
// point to catch a run created between plan-build and execution
|
||||
// referencing this path. Re-running the whole org-wide validation
|
||||
// again here, per bos target, made a project with a few thousand
|
||||
// BOS targets take hours (each call re-fetches and re-scans every
|
||||
// other run in the organization).
|
||||
browser, conn, cleanup, err := s.resolveObjectStorageBrowser(ctx, t.ConnectionID)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -641,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s
|
||||
}
|
||||
}
|
||||
}
|
||||
runtimeDone := false
|
||||
for index := range task.Plan.Targets {
|
||||
target := &task.Plan.Targets[index]
|
||||
if target.Status == "done" {
|
||||
continue
|
||||
targetConcurrency := 12
|
||||
if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" {
|
||||
if n, e := strconv.Atoi(v); e == nil && n > 0 {
|
||||
targetConcurrency = n
|
||||
}
|
||||
if target.Kind != "runtime" && !runtimeDone {
|
||||
deadline := time.Now().Add(150 * time.Second)
|
||||
for {
|
||||
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
busy := false
|
||||
for _, id := range task.Plan.RunIDs {
|
||||
run, e := s.store.GetRun(ctx, id)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
if run != nil && run.ExecutionLeaseOwner != "" {
|
||||
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
|
||||
if e != nil || until.After(time.Now()) {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := range items {
|
||||
item := &items[i]
|
||||
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
w := mapValue(item.Subject.Payload["sliceWorkflow"])
|
||||
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fail(ctx.Err())
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
// processBatch runs deleteProjectTarget for the not-yet-done targets at
|
||||
// indices with up to targetConcurrency in flight at once. Each target's
|
||||
// Status/Error and every persist() call is guarded by mu, so concurrent
|
||||
// workers never race on shared task state; deleteProjectTarget itself
|
||||
// (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs
|
||||
// unlocked. A single project's deletion used to process every target
|
||||
// strictly one at a time, which made stopping thousands of real runs
|
||||
// take as long as thousands of sequential K8s round trips.
|
||||
var mu sync.Mutex
|
||||
processBatch := func(indices []int) error {
|
||||
sem := make(chan struct{}, targetConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
var firstErr error
|
||||
for _, index := range indices {
|
||||
mu.Lock()
|
||||
skip := task.Plan.Targets[index].Status == "done" || firstErr != nil
|
||||
mu.Unlock()
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
// Catch work admitted immediately before the project lock.
|
||||
for _, step := range task.Plan.Targets {
|
||||
if step.Kind == "runtime" {
|
||||
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
mu.Lock()
|
||||
if firstErr != nil {
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
// Scoped exactly as before (projectId match only, no workflowId
|
||||
// fallback — see the loop below) but via an indexed query
|
||||
// instead of an organization-wide scan.
|
||||
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
|
||||
target := &task.Plan.Targets[index]
|
||||
target.Error = ""
|
||||
target.Status = "running"
|
||||
persistErr := persist()
|
||||
snapshot := *target
|
||||
mu.Unlock()
|
||||
if persistErr != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = persistErr
|
||||
}
|
||||
mu.Unlock()
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot)
|
||||
mu.Lock()
|
||||
target = &task.Plan.Targets[index]
|
||||
if runErr != nil {
|
||||
target.Status = "failed"
|
||||
target.Error = runErr.Error()
|
||||
if firstErr == nil {
|
||||
firstErr = runErr
|
||||
}
|
||||
} else {
|
||||
target.Status = "done"
|
||||
}
|
||||
_ = persist()
|
||||
mu.Unlock()
|
||||
}(index)
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
indicesByKind := map[string][]int{}
|
||||
nonRuntimePending := false
|
||||
for index, t := range task.Plan.Targets {
|
||||
indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index)
|
||||
if t.Kind != "runtime" && t.Status != "done" {
|
||||
nonRuntimePending = true
|
||||
}
|
||||
}
|
||||
if e := processBatch(indicesByKind["runtime"]); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
// Re-run the phase-transition safety checks (wait for in-flight human
|
||||
// work, re-verify runtime absence, revalidate references) exactly once
|
||||
// per execution attempt, right before touching any file/storage target
|
||||
// — the same point the original sequential loop ran them at.
|
||||
if nonRuntimePending {
|
||||
deadline := time.Now().Add(150 * time.Second)
|
||||
for {
|
||||
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
known := map[string]bool{}
|
||||
busy := false
|
||||
for _, id := range task.Plan.RunIDs {
|
||||
known[id] = true
|
||||
run, e := s.store.GetRun(ctx, id)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
if run != nil && run.ExecutionLeaseOwner != "" {
|
||||
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
|
||||
if e != nil || until.After(time.Now()) {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range fresh {
|
||||
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
|
||||
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
|
||||
for i := range items {
|
||||
item := &items[i]
|
||||
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
w := mapValue(item.Subject.Payload["sliceWorkflow"])
|
||||
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fail(ctx.Err())
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
// Catch work admitted immediately before the project lock.
|
||||
for _, step := range task.Plan.Targets {
|
||||
if step.Kind == "runtime" {
|
||||
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
runtimeDone = true
|
||||
}
|
||||
target.Error = ""
|
||||
target.Status = "running"
|
||||
if e := persist(); e != nil {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil {
|
||||
target.Status = "failed"
|
||||
target.Error = e.Error()
|
||||
// Scoped exactly as before (projectId match only, no workflowId
|
||||
// fallback — see the loop below) but via an indexed query
|
||||
// instead of an organization-wide scan.
|
||||
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
target.Status = "done"
|
||||
if e := persist(); e != nil {
|
||||
cancel()
|
||||
known := map[string]bool{}
|
||||
for _, id := range task.Plan.RunIDs {
|
||||
known[id] = true
|
||||
}
|
||||
for _, r := range fresh {
|
||||
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
|
||||
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
|
||||
return
|
||||
}
|
||||
}
|
||||
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"workspace", "bos", "records"} {
|
||||
if e := processBatch(indicesByKind[kind]); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) {
|
||||
t.Fatal("raw data removed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the
|
||||
// processBatch concurrency added to executeProjectDeletion: with many runs
|
||||
// (many runtime + workspace targets processed by a worker pool instead of
|
||||
// one at a time), the task must still end up "completed" with every target
|
||||
// "done" and every run's workspace directory actually removed — i.e. the
|
||||
// concurrent rewrite produces the same end state as the old sequential
|
||||
// loop, just faster.
|
||||
func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := store.NewMemoryStore()
|
||||
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s := &server{store: db, workspaceMountPath: t.TempDir()}
|
||||
const runCount = 30
|
||||
dirs := make([]string, 0, runCount)
|
||||
for i := 0; i < runCount; i++ {
|
||||
runID := fmt.Sprintf("run-concurrent-fixture-%02d", i)
|
||||
if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
dir, e := safeRunWorkspace(s.workspaceMountPath, runID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = os.MkdirAll(dir, 0700); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
plan, e := s.buildProjectDeletionPlan(ctx, p)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(plan.Blockers) > 0 {
|
||||
t.Fatal(plan.Blockers)
|
||||
}
|
||||
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
old := projectDeletionRuntimeFactory
|
||||
defer func() { projectDeletionRuntimeFactory = old }()
|
||||
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
|
||||
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
|
||||
}
|
||||
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.executeProjectDeletion(*claimed, "owner")
|
||||
finished, _ := db.GetProjectDeletion(ctx, task.ID)
|
||||
if finished.Status != "completed" {
|
||||
t.Fatalf("%+v", finished)
|
||||
}
|
||||
for _, target := range finished.Plan.Targets {
|
||||
if target.Status != "done" {
|
||||
t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target)
|
||||
}
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if _, e = os.Stat(dir); !os.IsNotExist(e) {
|
||||
t.Fatalf("workspace %s not removed", dir)
|
||||
}
|
||||
}
|
||||
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
|
||||
t.Fatal("project remains")
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,15 @@ import (
|
||||
)
|
||||
|
||||
func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Request, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, batch *domain.HumanBatchRecord, payload annotationReviewInput) {
|
||||
returnAuthor := ""
|
||||
if payload.Decision == domain.AnnotationReviewRejected {
|
||||
var err error
|
||||
returnAuthor, err = s.reworkAuthor(r.Context(), item)
|
||||
if err != nil {
|
||||
renderError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
op := operationFromRequest(r)
|
||||
now := op.CreatedAt
|
||||
beforeTask, beforeItem := task, item
|
||||
@ -26,6 +35,10 @@ func (s *server) planQualityReviewDecision(w http.ResponseWriter, r *http.Reques
|
||||
item.ReviewRound++
|
||||
item.CurrentStageID = "annotation_combined"
|
||||
item.Status = domain.HumanItemRework
|
||||
item.AssignedUserID, task.AssignedUserID = returnAuthor, returnAuthor
|
||||
if returnAuthor != "" {
|
||||
item.ClaimedAt = now
|
||||
}
|
||||
if item.ReviewRound > item.MaxReworkRounds {
|
||||
item.Status = domain.HumanItemEscalated
|
||||
task.Status = domain.AnnotationTaskStatusRejected
|
||||
|
||||
41
internal/api/rework_assignment.go
Normal file
41
internal/api/rework_assignment.go
Normal file
@ -0,0 +1,41 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Resolve before recording a decision: transient lookup failures must not silently
|
||||
// turn a return-to-author decision into a public-pool assignment.
|
||||
func (s *server) reworkAuthor(ctx context.Context, item domain.HumanItemRecord) (string, error) {
|
||||
if item.LatestAuthorUserID == "" || item.ReviewRound+1 > item.MaxReworkRounds {
|
||||
return "", nil
|
||||
}
|
||||
user, err := s.store.GetUser(ctx, item.LatestAuthorUserID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if user == nil || user.OrganizationID != item.OrganizationID || user.Status != domain.UserStatusActive || user.Role != domain.UserRoleAnnotator || user.SupplierID == "" {
|
||||
return "", nil
|
||||
}
|
||||
if item.SupplierID != "" && item.SupplierID != user.SupplierID {
|
||||
return "", nil
|
||||
}
|
||||
supplier, err := s.store.GetSupplier(ctx, user.SupplierID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if supplier == nil || supplier.OrganizationID != item.OrganizationID || !strings.EqualFold(supplier.Status, "active") {
|
||||
return "", nil
|
||||
}
|
||||
members, err := s.annotationMembershipsForUser(ctx, item.OrganizationID, item.ProjectID, user.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
task := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, ProcessingScope: item.ProcessingScope}
|
||||
if !membershipHasRole(members, task, domain.AnnotationRoleAnnotator) {
|
||||
return "", nil
|
||||
}
|
||||
return user.ID, nil
|
||||
}
|
||||
85
internal/api/rework_assignment_test.go
Normal file
85
internal/api/rework_assignment_test.go
Normal file
@ -0,0 +1,85 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQualityRejectionReturnsToEligibleAuthor(t *testing.T) {
|
||||
for _, durable := range []bool{false, true} {
|
||||
for _, scenario := range []string{"eligible", "disabled", "wrong-supplier", "no-membership", "supplier-inactive", "no-author", "escalated"} {
|
||||
t.Run(scenario+map[bool]string{true: "-durable", false: "-legacy"}[durable], func(t *testing.T) {
|
||||
f := decisionFixture(t)
|
||||
f.item.LatestAuthorUserID = f.annotator.ID
|
||||
f.item.SupplierID = f.annotator.SupplierID
|
||||
f.task.SupplierID = f.annotator.SupplierID
|
||||
f.item.MaxReworkRounds = 5
|
||||
switch scenario {
|
||||
case "disabled":
|
||||
f.annotator.Status = domain.UserStatusDisabled
|
||||
f.store.UpsertUser(f.ctx, f.annotator)
|
||||
case "wrong-supplier":
|
||||
f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: "other", OrganizationID: f.item.OrganizationID, Name: "Other", Status: "active"})
|
||||
f.annotator.SupplierID = "other"
|
||||
f.annotator.ID = "different-supplier-author"
|
||||
f.item.LatestAuthorUserID = f.annotator.ID
|
||||
f.store.UpsertUser(f.ctx, f.annotator)
|
||||
case "no-membership":
|
||||
f.store.DeleteProjectMember(f.ctx, "member-"+f.annotator.ID)
|
||||
case "supplier-inactive":
|
||||
f.store.UpsertSupplier(f.ctx, domain.SupplierRecord{ID: f.annotator.SupplierID, OrganizationID: f.item.OrganizationID, Name: "Inactive", Status: "inactive"})
|
||||
case "no-author":
|
||||
f.item.LatestAuthorUserID = ""
|
||||
case "escalated":
|
||||
f.item.MaxReworkRounds = 1
|
||||
}
|
||||
f.store.UpsertHumanItem(f.ctx, f.item)
|
||||
f.store.UpsertAnnotationTask(f.ctx, f.task)
|
||||
input := map[string]any{"stage": "quality", "decision": "rejected", "reasonCodes": []string{"manual_review_reject"}, "comment": "请修正", "problemSliceIds": []string{"left"}}
|
||||
endpoint := "/api/plugins/annotation-qa/tasks/" + f.task.ID + "/reviews"
|
||||
var code int
|
||||
var body string
|
||||
if durable {
|
||||
version, headers := openReviewDraftSession(t, f)
|
||||
input["operationId"] = "return-author"
|
||||
input["draftVersion"] = 0
|
||||
input["resultVersion"] = version
|
||||
r := reviewSessionRequest(f, http.MethodPost, endpoint, f.reviewer, input, headers)
|
||||
code = r.Code
|
||||
body = r.Body.String()
|
||||
} else {
|
||||
r := f.request(http.MethodPost, endpoint, f.reviewer, input)
|
||||
code = r.Code
|
||||
body = r.Body.String()
|
||||
}
|
||||
if code != 201 {
|
||||
t.Fatalf("review %d %s", code, body)
|
||||
}
|
||||
item, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
task, _ := f.store.GetAnnotationTask(f.ctx, f.task.ID)
|
||||
expected := ""
|
||||
if scenario == "eligible" {
|
||||
expected = f.annotator.ID
|
||||
}
|
||||
if item.AssignedUserID != expected || task.AssignedUserID != expected {
|
||||
t.Fatalf("return owner item=%q task=%q want=%q", item.AssignedUserID, task.AssignedUserID, expected)
|
||||
}
|
||||
if item.CurrentRevision != f.item.CurrentRevision {
|
||||
t.Fatal("annotation revision changed")
|
||||
}
|
||||
if expected != "" {
|
||||
r := f.request(http.MethodGet, "/api/human-items?pool=mine&projectId="+item.ProjectID, f.annotator, nil)
|
||||
var p struct {
|
||||
Items []domain.HumanItemRecord `json:"items"`
|
||||
}
|
||||
json.Unmarshal(r.Body.Bytes(), &p)
|
||||
if len(p.Items) != 1 {
|
||||
t.Fatalf("returned task missing in author's pool: %s", r.Body.String())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -10449,7 +10449,7 @@ func TestObjectStorageContentSupportsRangeAndHeadWithoutReadingWholeObject(t *te
|
||||
if item == nil || item.ID != connection.ID {
|
||||
t.Fatalf("unexpected storage connection: %#v", item)
|
||||
}
|
||||
return fakeBrowser, func() {}, nil
|
||||
return &unsignedVideoBrowser{fakeBrowser}, func() {}, nil
|
||||
}
|
||||
defer func() { storageConnectionBrowserResolver = originalResolver }()
|
||||
|
||||
|
||||
86
internal/api/video_playback_redirect_test.go
Normal file
86
internal/api/video_playback_redirect_test.go
Normal file
@ -0,0 +1,86 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
// A backend without presigning still supports the existing Range proxy.
|
||||
type unsignedVideoBrowser struct{ *fakeObjectStorageBrowser }
|
||||
|
||||
func (b *unsignedVideoBrowser) BuildPreviewURL(context.Context, string, string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func TestVideoPlaybackRedirectsRangesWithoutProxyingBytes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := store.NewMemoryStore()
|
||||
connection, err := data.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
||||
Name: "video-test", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test",
|
||||
Bucket: "videos", AccessKeyID: "test-ak", SecretAccessKey: "test-sk",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
browser := newFakeObjectStorageBrowser()
|
||||
key := "videos|capture.mp4"
|
||||
browser.objects[key] = domain.ObjectStorageObjectSummary{
|
||||
Bucket: "videos", Key: "capture.mp4", Name: "capture.mp4", Kind: "video",
|
||||
SizeBytes: 10, ContentType: "video/mp4",
|
||||
}
|
||||
browser.content[key] = []byte("0123456789")
|
||||
old := storageConnectionBrowserResolver
|
||||
defer func() { storageConnectionBrowserResolver = old }()
|
||||
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
return browser, nil, nil
|
||||
}
|
||||
handler := NewHandler(data, nil)
|
||||
path := "/api/object-storage/content?connectionId=" + url.QueryEscape(connection.ID) + "&bucket=videos&key=capture.mp4"
|
||||
for _, byteRange := range []string{"bytes=0-", "bytes=2-5", "bytes=-4", ""} {
|
||||
request := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
request.Header.Set("Range", byteRange)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusTemporaryRedirect {
|
||||
t.Fatalf("range %q: status %d, want 307", byteRange, response.Code)
|
||||
}
|
||||
if response.Header().Get("Location") != "https://example.com/preview/videos/capture.mp4" {
|
||||
t.Fatal("expected presigned storage target")
|
||||
}
|
||||
if response.Header().Get("Cache-Control") != "private, no-store" {
|
||||
t.Fatal("signed redirects must not be cached")
|
||||
}
|
||||
}
|
||||
if browser.readCalls[key] != 0 || browser.readRangeCalls[key] != 0 {
|
||||
t.Fatal("API must not transfer any video bytes")
|
||||
}
|
||||
// Invalid ranges remain rejected locally; metadata-only reads remain HEAD.
|
||||
for _, tc := range []struct {
|
||||
method, byteRange string
|
||||
status int
|
||||
}{
|
||||
{http.MethodGet, "bytes=20-", 416}, {http.MethodHead, "", 200},
|
||||
} {
|
||||
request := httptest.NewRequest(tc.method, path, nil)
|
||||
request.Header.Set("Range", tc.byteRange)
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != tc.status || response.Header().Get("Location") != "" {
|
||||
t.Fatalf("method %s range %q: status %d", tc.method, tc.byteRange, response.Code)
|
||||
}
|
||||
}
|
||||
// The original authorization gate is still in front of URL issuance.
|
||||
t.Setenv("DROBOTICFLOW_REQUIRE_SESSION_AUTH", "true")
|
||||
authenticatedHandler := NewHandler(data, nil)
|
||||
response := httptest.NewRecorder()
|
||||
authenticatedHandler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
if response.Code != http.StatusUnauthorized || response.Header().Get("Location") != "" {
|
||||
t.Fatalf("anonymous request leaked video URL: status %d", response.Code)
|
||||
}
|
||||
}
|
||||
@ -872,6 +872,12 @@ type RunRecord struct {
|
||||
Error string `json:"error,omitempty" bson:"error,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
// ArchivedAt marks a terminal run excluded from org-wide scans (e.g. the
|
||||
// deletion-safety reference check) once it's old enough that its data no
|
||||
// longer needs to participate in day-to-day queries. The record itself is
|
||||
// untouched — archiving never deletes it; only deleting the run's own
|
||||
// project does that.
|
||||
ArchivedAt string `json:"archivedAt,omitempty" bson:"archivedAt,omitempty"`
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
|
||||
@ -84,7 +84,7 @@ func (store *MongoStore) updateAdministrator(ctx context.Context, item domain.Us
|
||||
_, e = store.users.UpdateOne(sc, immutableSupplierFilter(item.ID, item.SupplierID), bson.M{"$set": fields})
|
||||
return nil, e
|
||||
})
|
||||
if mongo.IsDuplicateKeyError(err) {
|
||||
if isPhoneUniqueDuplicateKeyError(err) {
|
||||
return ErrDuplicateUserPhone
|
||||
}
|
||||
return err
|
||||
|
||||
@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -14,22 +16,60 @@ import (
|
||||
|
||||
var ErrAccountMutationBusy = errors.New("account mutation is busy or requires recovery")
|
||||
|
||||
// A durable, per-resource mutex also works on standalone MongoDB. It deliberately
|
||||
// has no expiring lease: a paused process must never resume after another writer
|
||||
// has stolen its lock. After a process crash or ambiguous network result, an
|
||||
// operator must verify that owner has stopped and inspect the affected resource
|
||||
// before clearing that exact owner's lock. Normal exits release their own lock.
|
||||
// accountMutationLockLeaseDuration bounds how long a held lock survives its
|
||||
// owner going silent (crash, pod replacement, OOM) before another process
|
||||
// may reclaim it. Renewed well before expiry by a background ticker while
|
||||
// the guarded operation is still running (see withAccountMutationLock), so
|
||||
// a live owner's lease never lapses out from under it — only a dead one's
|
||||
// does. Configurable because "how long is safe to wait before auto-
|
||||
// reclaiming" is an operational judgment call, not a constant.
|
||||
func accountMutationLockLeaseDuration() time.Duration {
|
||||
const fallback = 2 * time.Minute
|
||||
raw := strings.TrimSpace(os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS"))
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
seconds, err := strconv.Atoi(raw)
|
||||
if err != nil || seconds <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
// A durable, per-resource mutex also works on standalone MongoDB. It carries
|
||||
// an expiring lease (accountMutationLockLeaseDuration) renewed periodically
|
||||
// by a background ticker for as long as the guarded operation keeps running,
|
||||
// so a live owner's lock is never reclaimed out from under it — only a lock
|
||||
// whose owner stopped renewing (crashed, was replaced, or otherwise went
|
||||
// silent) becomes reclaimable, once its lease lapses. Reclaiming a lapsed
|
||||
// lease always assigns a fresh owner id, so the original holder's own
|
||||
// release (guarded by "owner": owner) can never match afterward, and never
|
||||
// touches the new holder's lock — the fresh id already acts as the fencing
|
||||
// token a naive TTL-lock would be missing. A crash or ambiguous network
|
||||
// result during the guarded write is still surfaced for the caller to
|
||||
// verify, exactly as before; only the "must wait for an operator to notice
|
||||
// and clear it by hand" half of the old design is gone.
|
||||
func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, operation func(context.Context) (any, error)) (result any, err error) {
|
||||
col := s.db.Collection("accountMutationLocks")
|
||||
_, e := col.InsertOne(ctx, bson.M{"_id": key, "locked": false})
|
||||
if e != nil && !mongo.IsDuplicateKeyError(e) {
|
||||
return nil, e
|
||||
}
|
||||
leaseDuration := accountMutationLockLeaseDuration()
|
||||
owner := domain.NewID("mutation")
|
||||
waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
for {
|
||||
r, e := col.UpdateOne(waitCtx, bson.M{"_id": key, "locked": false}, bson.M{"$set": bson.M{"locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"), "startedAt": time.Now().UTC()}})
|
||||
now := time.Now().UTC()
|
||||
filter := bson.M{"_id": key, "$or": bson.A{
|
||||
bson.M{"locked": false},
|
||||
bson.M{"locked": true, "leaseExpiresAt": bson.M{"$lt": now}},
|
||||
}}
|
||||
update := bson.M{"$set": bson.M{
|
||||
"locked": true, "owner": owner, "process": os.Getenv("HOSTNAME"),
|
||||
"startedAt": now, "leaseExpiresAt": now.Add(leaseDuration),
|
||||
}}
|
||||
r, e := col.UpdateOne(waitCtx, filter, update)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
@ -42,14 +82,52 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
// Renew well inside the lease window (a fixed fraction of it) so normal
|
||||
// scheduling jitter never lets a live owner's lease lapse; only an
|
||||
// owner that stops renewing entirely — because it's gone — loses the
|
||||
// lock. Mirrors the project-deletion task lease's own renewal ticker.
|
||||
renewCtx, cancelRenew := context.WithCancel(context.Background())
|
||||
renewalDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(renewalDone)
|
||||
ticker := time.NewTicker(leaseDuration / 4)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-renewCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
renewalCtx, renewalCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
r, e := col.UpdateOne(renewalCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"leaseExpiresAt": time.Now().UTC().Add(leaseDuration)}})
|
||||
renewalCancel()
|
||||
if e != nil {
|
||||
// A transient error (network blip, timeout) — not proof
|
||||
// the lock was actually lost. Keep the lease as-is and
|
||||
// retry on the next tick; giving up here on a live owner
|
||||
// would reintroduce the exact "two concurrent writers"
|
||||
// race the lease is meant to prevent.
|
||||
continue
|
||||
}
|
||||
if r.MatchedCount != 1 {
|
||||
// The document no longer matches our owner: someone else
|
||||
// genuinely reclaimed it after our lease lapsed. Stop
|
||||
// renewing; the release below will correctly no-op or
|
||||
// surface the conflict.
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() {
|
||||
cancelRenew()
|
||||
<-renewalDone
|
||||
// Do not unlock an uncertain write and admit a conflicting operation.
|
||||
if ctx.Err() != nil || mongo.IsNetworkError(err) || mongo.IsTimeout(err) {
|
||||
return
|
||||
}
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer releaseCancel()
|
||||
r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": ""}})
|
||||
r, e := col.UpdateOne(releaseCtx, bson.M{"_id": key, "owner": owner, "locked": true}, bson.M{"$set": bson.M{"locked": false}, "$unset": bson.M{"owner": "", "process": "", "startedAt": "", "leaseExpiresAt": ""}})
|
||||
if e != nil || r.MatchedCount != 1 {
|
||||
err = fmt.Errorf("%w: operation finished but lock release needs verification", ErrAccountMutationBusy)
|
||||
}
|
||||
|
||||
145
internal/store/account_mutation_lock_mongo_test.go
Normal file
145
internal/store/account_mutation_lock_mongo_test.go
Normal file
@ -0,0 +1,145 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// This test deliberately uses MongoDB, not MemoryStore: the lease/reclaim
|
||||
// behavior is a persisted-document race the in-memory store can't
|
||||
// reproduce. Found live on Fanyu: a lock whose owning process was replaced
|
||||
// mid-hold (a rolling worker deploy, or a crash) stayed "locked" forever —
|
||||
// the original design had no expiring lease by intent, trading availability
|
||||
// for a hand-verified recovery. This locks in the replacement: an expiring,
|
||||
// renewed lease that only a genuinely dead owner ever loses.
|
||||
func TestAccountMutationLockLeaseReclaimAndRenewal(t *testing.T) {
|
||||
uri := os.Getenv("MONGO_INDEX_TEST_URI")
|
||||
if uri == "" {
|
||||
t.Skip("MONGO_INDEX_TEST_URI required for real Mongo lock regression")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(ctx)
|
||||
db := os.Getenv("MONGO_INDEX_TEST_DB")
|
||||
if db == "" {
|
||||
t.Fatal("MONGO_INDEX_TEST_DB required")
|
||||
}
|
||||
dbHandle := client.Database(db)
|
||||
defer dbHandle.Collection("accountMutationLocks").Drop(context.Background())
|
||||
s := &MongoStore{db: dbHandle}
|
||||
|
||||
t.Run("normal acquire and release round-trips and unblocks the next waiter", func(t *testing.T) {
|
||||
key := "lease-test-normal"
|
||||
_, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "first", nil })
|
||||
if err != nil {
|
||||
t.Fatalf("first acquire: %v", err)
|
||||
}
|
||||
var doc bson.M
|
||||
if err := dbHandle.Collection("accountMutationLocks").FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doc["locked"] != false {
|
||||
t.Fatalf("expected lock released after normal completion, got %#v", doc)
|
||||
}
|
||||
if _, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) { return "second", nil }); err != nil {
|
||||
t.Fatalf("second acquire after release: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a lease that lapsed because its owner went silent is reclaimable", func(t *testing.T) {
|
||||
key := "lease-test-lapsed"
|
||||
col := dbHandle.Collection("accountMutationLocks")
|
||||
// Simulate a crashed holder: locked, with a lease already in the past.
|
||||
if _, err := col.InsertOne(ctx, bson.M{
|
||||
"_id": key, "locked": true, "owner": "mutation-dead-owner", "process": "dead-pod",
|
||||
"startedAt": time.Now().UTC().Add(-time.Hour), "leaseExpiresAt": time.Now().UTC().Add(-time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed := make(chan struct{})
|
||||
result, err := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) {
|
||||
close(claimed)
|
||||
return "reclaimed", nil
|
||||
})
|
||||
select {
|
||||
case <-claimed:
|
||||
default:
|
||||
t.Fatal("operation never ran — lapsed lease was not reclaimed")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("reclaim after lapse: %v", err)
|
||||
}
|
||||
if result != "reclaimed" {
|
||||
t.Fatalf("unexpected result: %v", result)
|
||||
}
|
||||
var doc bson.M
|
||||
if err := col.FindOne(ctx, bson.M{"_id": key}).Decode(&doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doc["locked"] != false {
|
||||
t.Fatalf("expected lock released after reclaim completed, got %#v", doc)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a live owner's lease is renewed before it can lapse, blocking a concurrent reclaim attempt", func(t *testing.T) {
|
||||
key := "lease-test-renewed"
|
||||
originalLease := os.Getenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS")
|
||||
os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", "1") // renew ticks every ~250ms
|
||||
defer func() {
|
||||
if originalLease == "" {
|
||||
os.Unsetenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS")
|
||||
} else {
|
||||
os.Setenv("ACCOUNT_MUTATION_LOCK_LEASE_SECONDS", originalLease)
|
||||
}
|
||||
}()
|
||||
|
||||
holderStarted := make(chan struct{})
|
||||
holderMayFinish := make(chan struct{})
|
||||
var reclaimAttempted atomic.Bool
|
||||
var reclaimErr error
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_, e := s.withAccountMutationLock(ctx, key, func(context.Context) (any, error) {
|
||||
close(holderStarted)
|
||||
<-holderMayFinish
|
||||
return "held", nil
|
||||
})
|
||||
if e != nil {
|
||||
t.Errorf("long-held operation returned an error: %v", e)
|
||||
}
|
||||
}()
|
||||
|
||||
<-holderStarted
|
||||
// Outlive several renewal ticks (lease=1s, renew every ~250ms) while
|
||||
// still holding the lock, then attempt a short-timeout reclaim: it
|
||||
// must fail, proving the live holder's lease never lapsed.
|
||||
time.Sleep(2500 * time.Millisecond)
|
||||
reclaimCtx, reclaimCancel := context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
_, reclaimErr = s.withAccountMutationLock(reclaimCtx, key, func(context.Context) (any, error) { return "stolen", nil })
|
||||
reclaimCancel()
|
||||
reclaimAttempted.Store(true)
|
||||
close(holderMayFinish)
|
||||
<-done
|
||||
|
||||
if !reclaimAttempted.Load() {
|
||||
t.Fatal("reclaim was never attempted")
|
||||
}
|
||||
if !errors.Is(reclaimErr, ErrAccountMutationBusy) {
|
||||
t.Fatalf("expected the renewed lease to block a concurrent reclaim, got: %v", reclaimErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
88
internal/store/account_phone_duplicate_error_mongo_test.go
Normal file
88
internal/store/account_phone_duplicate_error_mongo_test.go
Normal file
@ -0,0 +1,88 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex reproduces the exact
|
||||
// bug found live on Fanyu: a users collection that still carries a stale,
|
||||
// pre-phone-migration email_unique index (this codebase no longer declares
|
||||
// it, but an older database can still have it — see EnsureIndexes'
|
||||
// dropObsoleteIndexes call). A duplicate-key error from that unrelated index
|
||||
// must not be misreported as a phone conflict, or every account created
|
||||
// after the first one missing an email field fails with a "phone must be
|
||||
// unique" error that has nothing to do with the phone number entered.
|
||||
func TestPhoneUniqueDuplicateKeyErrorIgnoresUnrelatedIndex(t *testing.T) {
|
||||
uri := os.Getenv("MONGO_INDEX_TEST_URI")
|
||||
if uri == "" {
|
||||
t.Skip("MONGO_INDEX_TEST_URI required for real Mongo index regression")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(ctx)
|
||||
db := os.Getenv("MONGO_INDEX_TEST_DB")
|
||||
if db == "" {
|
||||
t.Fatal("MONGO_INDEX_TEST_DB required")
|
||||
}
|
||||
c := client.Database(db).Collection(fmt.Sprintf("users_phone_dup_test_%d", time.Now().UnixNano()))
|
||||
defer c.Drop(context.Background())
|
||||
|
||||
// A pre-migration-shaped users collection: phone_unique (current) plus a
|
||||
// leftover, non-sparse email_unique (obsolete, but not yet dropped).
|
||||
if _, err := c.Indexes().CreateMany(ctx, []mongo.IndexModel{
|
||||
uniqueIndex("phone_unique", bson.D{{Key: "phone", Value: 1}}),
|
||||
uniqueIndex("email_unique", bson.D{{Key: "email", Value: 1}}),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// First account has no email field — occupies email_unique's one null slot.
|
||||
if _, err := c.InsertOne(ctx, bson.M{"id": "user-a", "phone": "13800000001"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A second account, also missing email but with a genuinely distinct
|
||||
// phone, collides only with the obsolete email index — must NOT be
|
||||
// classified as a phone conflict.
|
||||
_, err = c.InsertOne(ctx, bson.M{"id": "user-b", "phone": "13800000002"})
|
||||
if !mongo.IsDuplicateKeyError(err) {
|
||||
t.Fatalf("expected a duplicate-key error from the stale email index, got: %v", err)
|
||||
}
|
||||
if isPhoneUniqueDuplicateKeyError(err) {
|
||||
t.Fatalf("email_unique conflict misclassified as a phone conflict: %v", err)
|
||||
}
|
||||
|
||||
// A third account reusing user-a's actual phone must still be correctly
|
||||
// classified as a real phone conflict.
|
||||
_, err = c.InsertOne(ctx, bson.M{"id": "user-c", "phone": "13800000001", "email": "user-c@example.com"})
|
||||
if !mongo.IsDuplicateKeyError(err) {
|
||||
t.Fatalf("expected a duplicate-key error from the phone index, got: %v", err)
|
||||
}
|
||||
if !isPhoneUniqueDuplicateKeyError(err) {
|
||||
t.Fatalf("real phone_unique conflict not recognized: %v", err)
|
||||
}
|
||||
|
||||
if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := dropObsoleteIndexes(ctx, c, "email_unique"); err != nil {
|
||||
t.Fatalf("dropping an already-absent index must be a no-op: %v", err)
|
||||
}
|
||||
|
||||
// With the obsolete index gone, a second account missing email succeeds.
|
||||
if _, err := c.InsertOne(ctx, bson.M{"id": "user-d", "phone": "13800000003"}); err != nil {
|
||||
t.Fatalf("account creation still blocked after dropping the obsolete index: %v", err)
|
||||
}
|
||||
}
|
||||
@ -1281,23 +1281,34 @@ func (store *MemoryStore) ListRunArtifacts(ctx context.Context, status string) (
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Excludes archived runs, matching MongoStore's ListRunArtifactsByOrganization
|
||||
// (see ArchiveTerminalRuns) — this is the org-wide scan behind
|
||||
// validateDeletionReferences' safety check.
|
||||
func (store *MemoryStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
|
||||
items, err := store.ListRunsByOrganization(ctx, organizationID, status)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hydrateRunArtifactsWhenNeeded(items)
|
||||
return items, nil
|
||||
kept := make([]domain.RunRecord, 0, len(items))
|
||||
for _, run := range items {
|
||||
if run.ArchivedAt == "" {
|
||||
kept = append(kept, run)
|
||||
}
|
||||
}
|
||||
hydrateRunArtifactsWhenNeeded(kept)
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// ListRunArtifactsForProject mirrors the Mongo implementation's scope: runs
|
||||
// whose projectId matches projectID directly, or whose workflowId is in
|
||||
// workflowIDs.
|
||||
// workflowIDs. Deliberately includes archived runs — a project's own
|
||||
// deletion must still clean up every run it ever had, archived or not.
|
||||
func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) {
|
||||
all, err := store.ListRunArtifactsByOrganization(ctx, organizationID, "")
|
||||
all, err := store.ListRunsByOrganization(ctx, organizationID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hydrateRunArtifactsWhenNeeded(all)
|
||||
workflowSet := make(map[string]bool, len(workflowIDs))
|
||||
for _, id := range workflowIDs {
|
||||
workflowSet[id] = true
|
||||
@ -1311,6 +1322,29 @@ func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organi
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
|
||||
func (store *MemoryStore) ArchiveTerminalRuns(_ context.Context, olderThan time.Time) (int, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
threshold := olderThan.UTC().Format(time.RFC3339Nano)
|
||||
archived := 0
|
||||
for id, run := range store.runs {
|
||||
if run.ArchivedAt != "" {
|
||||
continue
|
||||
}
|
||||
if run.Status != domain.RunStatusCompleted && run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled {
|
||||
continue
|
||||
}
|
||||
if run.EndedAt == "" || run.EndedAt >= threshold {
|
||||
continue
|
||||
}
|
||||
run.ArchivedAt = nowISO()
|
||||
store.runs[id] = run
|
||||
archived++
|
||||
}
|
||||
return archived, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunStatus]int, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
@ -1211,3 +1211,102 @@ func TestListRunArtifactsForProjectScopesByProjectOrWorkflow(t *testing.T) {
|
||||
t.Fatalf("scope leaked unrelated runs: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveTerminalRunsOnlyArchivesOldTerminalUnarchivedRuns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
|
||||
old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano)
|
||||
recent := time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339Nano)
|
||||
threshold := time.Now().Add(-30 * 24 * time.Hour)
|
||||
|
||||
runs := []domain.RunRecord{
|
||||
{ID: "run-old-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old},
|
||||
{ID: "run-old-failed", OrganizationID: "org-1", Status: domain.RunStatusFailed, EndedAt: old},
|
||||
{ID: "run-old-cancelled", OrganizationID: "org-1", Status: domain.RunStatusCancelled, EndedAt: old},
|
||||
{ID: "run-recent-completed", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: recent},
|
||||
{ID: "run-old-running", OrganizationID: "org-1", Status: domain.RunStatusRunning, EndedAt: ""},
|
||||
{ID: "run-already-archived", OrganizationID: "org-1", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old},
|
||||
}
|
||||
for _, run := range runs {
|
||||
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
||||
t.Fatalf("upsert run %s: %v", run.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
archived, err := dataStore.ArchiveTerminalRuns(ctx, threshold)
|
||||
if err != nil {
|
||||
t.Fatalf("archive terminal runs: %v", err)
|
||||
}
|
||||
if archived != 3 {
|
||||
t.Fatalf("expected 3 runs archived, got %d", archived)
|
||||
}
|
||||
|
||||
for _, id := range []string{"run-old-completed", "run-old-failed", "run-old-cancelled"} {
|
||||
run, err := dataStore.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get run %s: %v", id, err)
|
||||
}
|
||||
if run.ArchivedAt == "" {
|
||||
t.Fatalf("expected %s to be archived", id)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"run-recent-completed", "run-old-running"} {
|
||||
run, err := dataStore.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get run %s: %v", id, err)
|
||||
}
|
||||
if run.ArchivedAt != "" {
|
||||
t.Fatalf("expected %s to remain unarchived, got %s", id, run.ArchivedAt)
|
||||
}
|
||||
}
|
||||
|
||||
// Re-running must not re-archive (and must not double count) an already
|
||||
// archived run or one that just got archived.
|
||||
archivedAgain, err := dataStore.ArchiveTerminalRuns(ctx, threshold)
|
||||
if err != nil {
|
||||
t.Fatalf("archive terminal runs (second pass): %v", err)
|
||||
}
|
||||
if archivedAgain != 0 {
|
||||
t.Fatalf("expected second pass to archive nothing, archived %d", archivedAgain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRunArtifactsByOrganizationExcludesArchivedRunsButForProjectStillIncludesThem(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
|
||||
old := time.Now().Add(-60 * 24 * time.Hour).UTC().Format(time.RFC3339Nano)
|
||||
runs := []domain.RunRecord{
|
||||
{ID: "run-active", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old},
|
||||
{ID: "run-archived", OrganizationID: "org-1", ProjectID: "project-a", Status: domain.RunStatusCompleted, EndedAt: old, ArchivedAt: old},
|
||||
}
|
||||
for _, run := range runs {
|
||||
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
||||
t.Fatalf("upsert run %s: %v", run.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
orgItems, err := dataStore.ListRunArtifactsByOrganization(ctx, "org-1", "")
|
||||
if err != nil {
|
||||
t.Fatalf("list run artifacts by organization: %v", err)
|
||||
}
|
||||
if len(orgItems) != 1 || orgItems[0].ID != "run-active" {
|
||||
t.Fatalf("expected only the unarchived run from the org-wide scan, got %#v", orgItems)
|
||||
}
|
||||
|
||||
// A project's own deletion planning must still see its archived runs —
|
||||
// archiving must never exempt a run from being cleaned up by its own
|
||||
// project's deletion.
|
||||
projectItems, err := dataStore.ListRunArtifactsForProject(ctx, "org-1", "project-a", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("list run artifacts for project: %v", err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, item := range projectItems {
|
||||
got[item.ID] = true
|
||||
}
|
||||
if !got["run-active"] || !got["run-archived"] {
|
||||
t.Fatalf("expected project scope to include both runs regardless of archive state, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1073,8 +1073,12 @@ func (store *MongoStore) ListRunArtifactsForProject(ctx context.Context, organiz
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Excludes archived runs: this is the query behind validateDeletionReferences'
|
||||
// org-wide safety scan, and an organization's run history is unbounded and
|
||||
// never otherwise shrinks, so a run old enough to be archived is excluded
|
||||
// here to keep that scan's cost from growing forever (see ArchiveTerminalRuns).
|
||||
func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
|
||||
filter := bson.M{}
|
||||
filter := bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}}
|
||||
if status != "" {
|
||||
filter["status"] = status
|
||||
}
|
||||
@ -1098,6 +1102,20 @@ func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, org
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
|
||||
func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) {
|
||||
filter := bson.M{
|
||||
"status": bson.M{"$in": bson.A{string(domain.RunStatusCompleted), string(domain.RunStatusFailed), string(domain.RunStatusCancelled)}},
|
||||
"endedAt": bson.M{"$ne": "", "$lt": olderThan.UTC().Format(time.RFC3339Nano)},
|
||||
"archivedAt": bson.M{"$in": bson.A{nil, ""}},
|
||||
}
|
||||
result, err := store.runs.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"archivedAt": nowISO()}})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(result.ModifiedCount), nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) {
|
||||
cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{
|
||||
{{Key: "$group", Value: bson.M{
|
||||
@ -2237,12 +2255,21 @@ func (store *MongoStore) upsertUser(ctx context.Context, item domain.UserRecord,
|
||||
return ErrUserMutationConflict
|
||||
}
|
||||
}
|
||||
if mongo.IsDuplicateKeyError(err) {
|
||||
if isPhoneUniqueDuplicateKeyError(err) {
|
||||
return ErrDuplicateUserPhone
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// isPhoneUniqueDuplicateKeyError reports whether err is a MongoDB duplicate-key
|
||||
// error specifically from the users collection's phone_unique index, not just
|
||||
// any unique-index conflict on that collection — a stale or unrelated index
|
||||
// (e.g. a pre-migration email_unique index left over in an older database)
|
||||
// would otherwise be misreported to the caller as "phone must be unique".
|
||||
func isPhoneUniqueDuplicateKeyError(err error) bool {
|
||||
return mongo.IsDuplicateKeyError(err) && strings.Contains(err.Error(), "phone_unique")
|
||||
}
|
||||
|
||||
func (store *MongoStore) CreateSession(ctx context.Context, item domain.SessionRecord) (*domain.SessionRecord, error) {
|
||||
if strings.TrimSpace(item.ID) == "" {
|
||||
item.ID = domain.NewID("session")
|
||||
|
||||
@ -2,6 +2,7 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
@ -118,6 +119,9 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
index("status_updatedAt", bson.D{{Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}),
|
||||
index("status_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
// Backs ArchiveTerminalRuns' periodic sweep for terminal, old,
|
||||
// not-yet-archived runs.
|
||||
index("status_endedAt", bson.D{{Key: "status", Value: 1}, {Key: "endedAt", Value: 1}}),
|
||||
index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("workflowId_taskId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}),
|
||||
@ -378,5 +382,27 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return dropObsoleteIndexes(ctx, store.users, "email_unique")
|
||||
}
|
||||
|
||||
// dropObsoleteIndexes removes indexes this codebase no longer declares. A
|
||||
// database created before the accounts moved from email to phone as the
|
||||
// sole identifier can still carry a non-sparse email_unique index (this
|
||||
// function never creates it — only pre-migration deployments have it); left
|
||||
// in place, the first account created without an email field occupies that
|
||||
// index's one allowed null slot, and every account created after that
|
||||
// mislabels as "phone must be unique" since the caller only checks whether
|
||||
// *some* unique index on the collection conflicted. Missing-index errors
|
||||
// are ignored since that's the desired end state.
|
||||
func dropObsoleteIndexes(ctx context.Context, collection *mongo.Collection, names ...string) error {
|
||||
for _, name := range names {
|
||||
if _, err := collection.Indexes().DropOne(ctx, name); err != nil {
|
||||
var cmdErr mongo.CommandError
|
||||
if errors.As(err, &cmdErr) && cmdErr.Code == 27 { // IndexNotFound
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -104,6 +104,14 @@ type Store interface {
|
||||
// every run in the organization (see account-phone-login-era perf fix:
|
||||
// large orgs made the unscoped org-wide scan hang for minutes).
|
||||
ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error)
|
||||
// ArchiveTerminalRuns sets ArchivedAt on every not-yet-archived run whose
|
||||
// status is completed/failed/cancelled and whose EndedAt is older than
|
||||
// olderThan. Archiving never deletes the record — it only excludes it
|
||||
// from the org-wide scans (see ListRunArtifactsByOrganization) that
|
||||
// would otherwise keep growing as an organization's run history grows
|
||||
// unbounded; a run's data is only actually removed when its project is
|
||||
// deleted. Returns the number of runs archived.
|
||||
ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error)
|
||||
CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error)
|
||||
ListRunSummaries(context.Context, string) ([]domain.RunSummary, error)
|
||||
ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error)
|
||||
|
||||
@ -99,6 +99,9 @@ type Runtime struct {
|
||||
lastOrphanCheckNano atomic.Int64
|
||||
pendingFailureAfter time.Duration
|
||||
missingMaxRecoveries int
|
||||
runArchiveMinimumAge time.Duration
|
||||
runArchiveInterval time.Duration
|
||||
lastRunArchiveNano atomic.Int64
|
||||
}
|
||||
|
||||
type ProcessOnceStatus string
|
||||
@ -203,6 +206,8 @@ func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName str
|
||||
orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute),
|
||||
pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute),
|
||||
missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3),
|
||||
runArchiveMinimumAge: envDurationSeconds("WORKER_RUN_ARCHIVE_MIN_AGE_SECONDS", 30*24*time.Hour),
|
||||
runArchiveInterval: envDurationSeconds("WORKER_RUN_ARCHIVE_CHECK_INTERVAL_SECONDS", time.Hour),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -1076,6 +1081,39 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
|
||||
runtime.metrics.RecordOrphanCleanup(cleaned, cleanupErrors)
|
||||
}
|
||||
|
||||
// archiveTerminalRuns periodically marks old, terminal runs as archived (see
|
||||
// domain.RunRecord.ArchivedAt) so an organization's ever-growing run history
|
||||
// doesn't keep inflating the cost of org-wide scans like
|
||||
// validateDeletionReferences' safety check. Throttled the same way
|
||||
// reconcileOrphanedRuntimes is, since this only needs to run occasionally,
|
||||
// not on every poll tick.
|
||||
func (runtime *Runtime) archiveTerminalRuns(ctx context.Context) {
|
||||
now := time.Now().UTC()
|
||||
interval := runtime.runArchiveInterval
|
||||
if interval <= 0 {
|
||||
interval = time.Hour
|
||||
}
|
||||
lastCheck := runtime.lastRunArchiveNano.Load()
|
||||
if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval {
|
||||
return
|
||||
}
|
||||
if !runtime.lastRunArchiveNano.CompareAndSwap(lastCheck, now.UnixNano()) {
|
||||
return
|
||||
}
|
||||
minimumAge := runtime.runArchiveMinimumAge
|
||||
if minimumAge <= 0 {
|
||||
minimumAge = 30 * 24 * time.Hour
|
||||
}
|
||||
archived, err := runtime.store.ArchiveTerminalRuns(ctx, now.Add(-minimumAge))
|
||||
if err != nil {
|
||||
fmt.Printf("[worker] archive terminal runs failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
if archived > 0 {
|
||||
fmt.Printf("[worker] archived %d terminal run(s) older than %s\n", archived, minimumAge)
|
||||
}
|
||||
}
|
||||
|
||||
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
||||
startedAt := time.Now()
|
||||
var processErr error
|
||||
@ -1138,6 +1176,7 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
||||
}
|
||||
runtime.applyWorkflowRuntimePolicies(ctx, workflows)
|
||||
runtime.reconcileOrphanedRuntimes(ctx)
|
||||
runtime.archiveTerminalRuns(ctx)
|
||||
queuedRuns := runRecordsFromSummaries(queuedPage.Items)
|
||||
selectedRunIDs := selectQueuedRunsForDispatchSnapshot(
|
||||
queuedRuns,
|
||||
|
||||
@ -6,6 +6,8 @@ metadata:
|
||||
namespace: __NAMESPACE__
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels: {app: droboticflow-annotation-processor}
|
||||
template:
|
||||
@ -22,8 +24,8 @@ spec:
|
||||
command: [/usr/local/bin/droboticflow-api]
|
||||
env:
|
||||
- {name: ANNOTATION_PROCESSOR_MODE, value: processor}
|
||||
- {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "2"}
|
||||
- {name: ANNOTATION_INDEX_BACKFILL, value: "1"}
|
||||
- {name: ANNOTATION_PROCESSOR_CONCURRENCY, value: "1"}
|
||||
- {name: ANNOTATION_INDEX_BACKFILL, value: "0"}
|
||||
- {name: WORKSPACE_MOUNT_PATH, value: "__WORKSPACE_MOUNT_PATH__"}
|
||||
- {name: GLOBAL_BOS_ENDPOINT, value: "__BOS_ENDPOINT__"}
|
||||
- {name: ANNOTATION_WORKSPACE_CONNECTION_ID, value: "__ANNOTATION_WORKSPACE_CONNECTION_ID__"}
|
||||
|
||||
@ -14,3 +14,6 @@ export WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
|
||||
export WORKSPACE_BOS_BUCKET="${WORKSPACE_BOS_BUCKET:-drobodata-fanyu}"
|
||||
export ANNOTATION_WORKSPACE_CONNECTION_ID="${ANNOTATION_WORKSPACE_CONNECTION_ID:-storage-d0a49fe4b9881c324f0640ab62359603}"
|
||||
export ANNOTATION_WORKSPACE_BUCKET="${ANNOTATION_WORKSPACE_BUCKET:-$WORKSPACE_BOS_BUCKET}"
|
||||
|
||||
# Keep heavy slice materialization out of the web API.
|
||||
export ANNOTATION_PROCESSOR_ENABLED="${ANNOTATION_PROCESSOR_ENABLED:-true}"
|
||||
|
||||
@ -379,8 +379,6 @@ export function CaptureAnnotationWorkbench({
|
||||
const [playbackRate, setPlaybackRate] = useState(1);
|
||||
const [showLeft, setShowLeft] = useState(initialCameraVisibility?.left ?? true);
|
||||
const [showRight, setShowRight] = useState(initialCameraVisibility?.right ?? Boolean(rightVideoSource));
|
||||
const effectiveShowLeft = focusedLayout || showLeft;
|
||||
const effectiveShowRight = focusedLayout ? Boolean(rightVideoSource) : showRight;
|
||||
const cameraStorageKey = cameraPreferenceKey
|
||||
? `${isQualityReview || providedReviewCaption ? "review" : "annotation"}-camera-default:${cameraPreferenceKey}`
|
||||
: "";
|
||||
@ -394,6 +392,7 @@ export function CaptureAnnotationWorkbench({
|
||||
};
|
||||
const [defaultCamera, setDefaultCamera] = useState(readCameraDefault);
|
||||
const initialInspection = () => {
|
||||
if (!rightVideoSource) return "left" as const;
|
||||
if (providedReviewCaption && initialCameraVisibility)
|
||||
return initialCameraVisibility.left && initialCameraVisibility.right
|
||||
? null
|
||||
@ -404,7 +403,11 @@ export function CaptureAnnotationWorkbench({
|
||||
return preference === "both" ? null : (preference as "left" | "right");
|
||||
};
|
||||
const [inspectionCamera, setInspectionCamera] = useState<"left" | "right" | null>(initialInspection);
|
||||
const effectiveShowLeft = focusedLayout ? inspectionCamera !== "right" || !rightVideoSource : showLeft;
|
||||
const effectiveShowRight = Boolean(rightVideoSource) && (focusedLayout ? inspectionCamera !== "left" : showRight);
|
||||
const inspectCamera = (camera: "left" | "right" | null) => {
|
||||
if (camera === inspectionCamera) return;
|
||||
transport.prepareCameraChange();
|
||||
setInspectionCamera(camera);
|
||||
if (providedReviewCaption) onCameraVisibilityChange?.({ left: camera !== "right", right: camera !== "left" });
|
||||
};
|
||||
@ -2650,7 +2653,7 @@ export function CaptureAnnotationWorkbench({
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={inspectionCamera === "left"}
|
||||
disabled={!effectiveShowLeft}
|
||||
disabled={!source}
|
||||
onClick={() => inspectCamera("left")}
|
||||
>
|
||||
放大左相机
|
||||
@ -2659,7 +2662,7 @@ export function CaptureAnnotationWorkbench({
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={inspectionCamera === "right"}
|
||||
disabled={!effectiveShowRight}
|
||||
disabled={!rightSource}
|
||||
onClick={() => inspectCamera("right")}
|
||||
>
|
||||
放大右相机
|
||||
@ -2944,7 +2947,17 @@ export function CaptureAnnotationWorkbench({
|
||||
)}
|
||||
{transport.mediaError && (
|
||||
<span role="alert" className="text-xs text-rose-300">
|
||||
{transport.mediaError}
|
||||
{transport.mediaError}{" "}
|
||||
<button
|
||||
type="button"
|
||||
className="underline"
|
||||
onClick={() => {
|
||||
transport.retry();
|
||||
focusWorkbench();
|
||||
}}
|
||||
>
|
||||
重新加载视频
|
||||
</button>
|
||||
</span>
|
||||
)}
|
||||
<span className="ml-auto inline-flex items-center gap-3">
|
||||
|
||||
@ -34,6 +34,27 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
const [isPositioning, setIsPositioning] = useState(false);
|
||||
const [mediaError, setMediaError] = useState("");
|
||||
const targets = useRef<{ video: HTMLVideoElement; time: number; revision: number }[]>([]);
|
||||
const retryTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const retryAttempts = useRef({ left: 0, right: 0 });
|
||||
const reloadSequence = useRef(0);
|
||||
|
||||
function cancelRetry() {
|
||||
if (retryTimer.current !== null) clearTimeout(retryTimer.current);
|
||||
retryTimer.current = null;
|
||||
}
|
||||
function reloadTrack(video: HTMLVideoElement) {
|
||||
const c = config.current;
|
||||
const source = video === c.leftRef.current ? c.leftSource : c.rightSource;
|
||||
if (!source) return;
|
||||
const url = new URL(source, window.location.href);
|
||||
// Return to the original authenticated route, never to currentSrc (which
|
||||
// can be an expired BOS redirect). Do not alter external URL signatures.
|
||||
if (url.origin === window.location.origin) {
|
||||
url.searchParams.set("_mediaRetry", `${Date.now()}-${++reloadSequence.current}`);
|
||||
}
|
||||
video.src = url.href;
|
||||
video.load();
|
||||
}
|
||||
|
||||
function visibleTracks() {
|
||||
const c = config.current;
|
||||
@ -66,6 +87,10 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
}
|
||||
function seekTrack(target: (typeof targets.current)[number]) {
|
||||
const { video, time } = target;
|
||||
if (video.error) {
|
||||
reloadTrack(video);
|
||||
return;
|
||||
}
|
||||
if (video.readyState < 1) return; // loadedmetadata will finish this request.
|
||||
const next = physicalTarget(video, time);
|
||||
if (Math.abs(video.currentTime - next) > 0.000001) video.currentTime = next;
|
||||
@ -77,19 +102,24 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
({ video, time }) =>
|
||||
// HTMLMediaElement can quantize to microseconds. This tolerance is
|
||||
// only for readiness, never for the saved/visible annotation time.
|
||||
video.seeking || video.readyState < 2 || Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002,
|
||||
video.error ||
|
||||
video.seeking ||
|
||||
video.readyState < 2 ||
|
||||
Math.abs(video.currentTime - physicalTarget(video, time)) > 0.000002,
|
||||
)
|
||||
)
|
||||
return;
|
||||
const revision = targets.current[0].revision;
|
||||
targets.current = [];
|
||||
setMediaError("");
|
||||
retryAttempts.current = { left: 0, right: 0 };
|
||||
queue.complete(revision);
|
||||
}
|
||||
const [queue] = useState(() =>
|
||||
createMediaSeekQueue({
|
||||
dispatch(time: number, revision: number) {
|
||||
cancelRetry();
|
||||
setIsPositioning(true);
|
||||
setMediaError("");
|
||||
pauseElements();
|
||||
targets.current = visibleTracks().map(({ video, toMedia }) => ({ video, time: toMedia(time), revision }));
|
||||
targets.current.forEach(seekTrack);
|
||||
@ -105,6 +135,7 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
);
|
||||
useEffect(
|
||||
() => () => {
|
||||
cancelRetry();
|
||||
playing.current = false;
|
||||
queue.cancel();
|
||||
targets.current = [];
|
||||
@ -165,11 +196,25 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
function play() {
|
||||
playing.current = true;
|
||||
setIsPlaying(true);
|
||||
if (visibleTracks().some(({ video }) => video.error)) {
|
||||
retry();
|
||||
return;
|
||||
}
|
||||
if (!queue.isPending() && !scrubbing.current) playElements();
|
||||
}
|
||||
|
||||
function retry() {
|
||||
cancelRetry();
|
||||
retryAttempts.current = { left: 0, right: 0 };
|
||||
queue.cancel();
|
||||
queue.commit(position.current);
|
||||
}
|
||||
|
||||
// Sources/visibility, not playhead updates, own resource lifetime.
|
||||
useEffect(() => {
|
||||
cancelRetry();
|
||||
retryAttempts.current = { left: 0, right: 0 };
|
||||
setMediaError("");
|
||||
queue.cancel();
|
||||
targets.current = [];
|
||||
scrubbing.current = false;
|
||||
@ -191,6 +236,7 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
}
|
||||
queue.commit(position.current);
|
||||
return () => {
|
||||
cancelRetry();
|
||||
queue.cancel();
|
||||
targets.current = [];
|
||||
};
|
||||
@ -210,12 +256,28 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
onLoadedData: settle,
|
||||
onCanPlay: settle,
|
||||
onError(event: { currentTarget: HTMLVideoElement }) {
|
||||
if (!event.currentTarget.getAttribute("src")) return;
|
||||
if (
|
||||
!event.currentTarget.getAttribute("src") ||
|
||||
!visibleTracks().some(({ video }) => video === event.currentTarget)
|
||||
)
|
||||
return;
|
||||
cancelRetry();
|
||||
queue.cancel();
|
||||
targets.current = [];
|
||||
setIsPositioning(false);
|
||||
setMediaError("视频加载失败,请重新定位或重新打开任务。");
|
||||
pause();
|
||||
setMediaError("视频加载失败,当前编辑已保留。");
|
||||
// Keep play intent and the logical position, so recovery does not
|
||||
// remount the editor or jump to the beginning of the video.
|
||||
pauseElements();
|
||||
if (config.current.enabled && retryAttempts.current[camera] < 2) {
|
||||
const attempt = ++retryAttempts.current[camera];
|
||||
retryTimer.current = setTimeout(() => {
|
||||
retryTimer.current = null;
|
||||
queue.commit(position.current);
|
||||
}, attempt * 750);
|
||||
} else {
|
||||
pause();
|
||||
}
|
||||
},
|
||||
onPlay(event: { currentTarget: HTMLVideoElement }) {
|
||||
if (event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current)
|
||||
@ -224,7 +286,13 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
setIsPlaying(true);
|
||||
},
|
||||
onPause(event: { currentTarget: HTMLVideoElement }) {
|
||||
if (!event.currentTarget.paused || !isMaster(event.currentTarget) || queue.isPending() || scrubbing.current)
|
||||
if (
|
||||
!event.currentTarget.paused ||
|
||||
!isMaster(event.currentTarget) ||
|
||||
queue.isPending() ||
|
||||
scrubbing.current ||
|
||||
retryTimer.current !== null
|
||||
)
|
||||
return;
|
||||
pause();
|
||||
},
|
||||
@ -257,6 +325,7 @@ export function useCaptureVideoTransport(options: Options) {
|
||||
pause,
|
||||
pauseAtFrame,
|
||||
play,
|
||||
retry,
|
||||
isPlaying,
|
||||
isPositioning,
|
||||
mediaError,
|
||||
|
||||
44
src/lib/invalid-slice-consolidation.js
Normal file
44
src/lib/invalid-slice-consolidation.js
Normal file
@ -0,0 +1,44 @@
|
||||
// Candidate discovery only. The server revalidates explicit selections and
|
||||
// creates a new revision; viewing or marking an action invalid never merges.
|
||||
export function invalidSliceConsolidationCandidates(slices, actions) {
|
||||
const ordered = [...slices].sort((a, b) => a.start - b.start);
|
||||
return ordered.flatMap((current, index) => {
|
||||
const left = ordered[index - 1];
|
||||
const right = ordered[index + 1];
|
||||
if (
|
||||
!left ||
|
||||
!right ||
|
||||
[left, current, right].some((s) => s.deliveryDisposition === "discard") ||
|
||||
!current.name?.trim() ||
|
||||
left.name !== current.name ||
|
||||
right.name !== current.name ||
|
||||
left.end !== current.start ||
|
||||
current.end !== right.start
|
||||
)
|
||||
return [];
|
||||
const inner = actions
|
||||
.filter((a) => a.end > current.start && a.start < current.end)
|
||||
.sort((a, b) => a.start - b.start);
|
||||
let cursor = current.start;
|
||||
if (
|
||||
!inner.length ||
|
||||
!inner.every((a) => {
|
||||
if (!a.isInvalid || a.start !== cursor || a.end <= a.start || a.end > current.end) return false;
|
||||
cursor = a.end;
|
||||
return true;
|
||||
}) ||
|
||||
cursor !== current.end
|
||||
)
|
||||
return [];
|
||||
return [
|
||||
{
|
||||
id: current.id,
|
||||
name: current.name,
|
||||
start: left.start,
|
||||
end: right.end,
|
||||
invalidStart: current.start,
|
||||
invalidEnd: current.end,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
@ -1,3 +1,4 @@
|
||||
import { invalidSliceConsolidationCandidates } from "../lib/invalid-slice-consolidation.js";
|
||||
import { ReviewDecisionDialog } from "../components/ReviewDecisionDialog";
|
||||
import { useReviewContinuation } from "../hooks/useReviewContinuation";
|
||||
import { useReviewEntryIntent } from "../hooks/useReviewEntryIntent";
|
||||
@ -3157,12 +3158,32 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
setError("请保存所有动作描述或明确设为无效");
|
||||
return;
|
||||
}
|
||||
const candidates = invalidSliceConsolidationCandidates(sliceSegments, segments);
|
||||
const consolidateInvalidSliceIds: string[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
window.confirm(
|
||||
`“${candidate.name}”中 ${candidate.invalidStart.toFixed(3)}–${candidate.invalidEnd.toFixed(3)} 秒的无效动作单独占了一个切片。\n\n是否与前后同名切片合并为 ${candidate.start.toFixed(3)}–${candidate.end.toFixed(3)} 秒的完整切片?全部动作描述、无效标记及时间保持不变。\n确定:合并外层;取消:保留原划分。`,
|
||||
)
|
||||
)
|
||||
consolidateInvalidSliceIds.push(candidate.id);
|
||||
}
|
||||
if (
|
||||
candidates.length &&
|
||||
!window.confirm(
|
||||
consolidateInvalidSliceIds.length
|
||||
? `已选择 ${consolidateInvalidSliceIds.length} 处无损合并。提交后重新生成审核视频,原版本保留。确认提交质检?`
|
||||
: "保留原切片划分,确认提交质检?",
|
||||
)
|
||||
)
|
||||
return;
|
||||
const currentTaskId = detail.task.id;
|
||||
const token = beginAnnotationMutation(currentTaskId);
|
||||
if (!token) return;
|
||||
clearEditHistory();
|
||||
try {
|
||||
const submission = await annotationOperationsApi.submitCombined(detail.task.id, {
|
||||
consolidateInvalidSliceIds,
|
||||
version: detailRef.current?.humanItem?.version ?? detail.humanItem.version,
|
||||
sliceSegments: applySliceStatuses(
|
||||
sliceSegments.map((slice) => ({ ...slice, taskLabelId: "", taskLabel: "" })),
|
||||
|
||||
@ -178,6 +178,7 @@ export const annotationOperationsApi = {
|
||||
taskId: string,
|
||||
payload: {
|
||||
version: number;
|
||||
consolidateInvalidSliceIds?: string[];
|
||||
sliceSegments: SliceSegment[];
|
||||
sliceOperations?: SliceOperation[];
|
||||
segments: AnnotationSegment[];
|
||||
|
||||
170
tests/browser/video-playback-recovery.mjs
Normal file
170
tests/browser/video-playback-recovery.mjs
Normal file
@ -0,0 +1,170 @@
|
||||
// Local real-component/media regression. Synthetic video only; no API writes.
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { chromium } from "playwright";
|
||||
const base = process.env.VIDEO_TEST_URL || "http://127.0.0.1:5196";
|
||||
assert(["127.0.0.1", "localhost"].includes(new URL(base).hostname), "local fixtures only");
|
||||
const out = process.env.VIDEO_TEST_OUTPUT || "/tmp/presigned-video-tests";
|
||||
await fs.mkdir(out, { recursive: true });
|
||||
const file = out + "/sample.mp4";
|
||||
try {
|
||||
await fs.access(file);
|
||||
} catch {
|
||||
execFileSync("ffmpeg", [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=size=240x320:rate=30:duration=12",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
file,
|
||||
]);
|
||||
}
|
||||
const bytes = await fs.readFile(file);
|
||||
const browser = await chromium.launch({
|
||||
executablePath:
|
||||
process.env.PLAYWRIGHT_EXECUTABLE_PATH || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||
headless: true,
|
||||
args: ["--no-proxy-server"],
|
||||
});
|
||||
try {
|
||||
for (const mode of ["annotation", "quality", "acceptance"]) {
|
||||
const page = await browser.newPage({ viewport: { width: 1413, height: 900 } });
|
||||
page.setDefaultTimeout(15000);
|
||||
let fail = false,
|
||||
failedRequests = 0;
|
||||
const requests = [],
|
||||
errors = [];
|
||||
page.on("pageerror", (e) => errors.push(e.message));
|
||||
await page.route("**/api/**", async (route) => {
|
||||
const req = route.request(),
|
||||
u = new URL(req.url());
|
||||
assert.equal(req.method(), "GET");
|
||||
if (!u.pathname.endsWith(".mp4")) return route.fulfill({ json: {} });
|
||||
requests.push({ path: u.pathname, retry: u.searchParams.get("_mediaRetry") });
|
||||
if (fail && u.pathname.includes("left")) {
|
||||
failedRequests++;
|
||||
return route.abort("failed");
|
||||
}
|
||||
const match = /bytes=(\d+)-(\d*)/.exec(req.headers().range || "");
|
||||
const start = match ? Number(match[1]) : 0,
|
||||
end = match?.[2] ? Math.min(Number(match[2]), bytes.length - 1) : bytes.length - 1;
|
||||
return route.fulfill({
|
||||
status: match ? 206 : 200,
|
||||
contentType: "video/mp4",
|
||||
headers: {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Length": String(end - start + 1),
|
||||
...(match ? { "Content-Range": `bytes ${start}-${end}/${bytes.length}` } : {}),
|
||||
},
|
||||
body: bytes.subarray(start, end + 1),
|
||||
});
|
||||
});
|
||||
await page.route("**/__video_recovery", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<html><body><div id="root"></div><script type="module">
|
||||
import RefreshRuntime from '/@react-refresh';RefreshRuntime.injectIntoGlobalHook(window);window.$RefreshReg$=()=>{};window.$RefreshSig$=()=>type=>type;window.__vite_plugin_react_preamble_installed__=true;
|
||||
const React=await import('/node_modules/.vite/deps/react.js').then(m=>m.default||m);const D=await import('/node_modules/.vite/deps/react-dom_client.js');await import('/src/index.css');
|
||||
const {CaptureAnnotationWorkbench}=await import('/src/components/CaptureAnnotationWorkbench.tsx');
|
||||
const mode=${JSON.stringify(mode)};
|
||||
const props={taskId:'synthetic-video',videoSource:'/api/video-test/left.mp4',rightVideoSource:'/api/video-test/right.mp4',browserReadyMedia:true,sourceIsFullVideo:true,focusedLayout:true,clipStart:0,clipEnd:12,slices:[{id:'slice',name:'Synthetic',start:0,end:12}],segments:[{id:'a',start:0,end:12,description:'Original',source:'manual',confidence:1}],labels:[],statuses:{},phase:'annotate',continuousActions:true,frameBoundaries:Array.from({length:361},(_,i)=>i/30),isQualityReview:mode!=='annotation',reviewMode:mode==='acceptance'?'acceptance':'quality',...(mode==='acceptance'?{reviewCaption:{text:'Original'}}:{}),onChange:()=>{throw Error('Unexpected annotation write')}};
|
||||
(D.createRoot||D.default.createRoot)(document.getElementById('root')).render(React.createElement('div',{className:'annotation-operations-page','data-workspace-compact':true},React.createElement(CaptureAnnotationWorkbench,props)));
|
||||
</script></body></html>`,
|
||||
}),
|
||||
);
|
||||
await page.goto(base + "/__video_recovery", { waitUntil: "domcontentloaded" });
|
||||
const bench = page.getByTestId("capture-annotation-workbench"),
|
||||
left = page.locator('video[data-camera="left"]'),
|
||||
right = page.locator('video[data-camera="right"]');
|
||||
const settle = () =>
|
||||
page.waitForFunction(() => {
|
||||
const videos = [...document.querySelectorAll("video")].filter((v) => v.getAttribute("src"));
|
||||
return videos.length && videos.every((v) => v.readyState >= 2 && !v.seeking && !v.error);
|
||||
});
|
||||
await bench.waitFor();
|
||||
await settle();
|
||||
assert.equal(await right.getAttribute("src"), null, "single-camera view must detach the hidden stream");
|
||||
assert.equal(
|
||||
requests.filter((r) => r.path.includes("right")).length,
|
||||
0,
|
||||
"hidden camera must make no media request",
|
||||
);
|
||||
await bench.getByRole("button", { name: "播放", exact: true }).click();
|
||||
await page.waitForFunction(() => document.querySelector('video[data-camera="left"]').currentTime > 1);
|
||||
const beforeSameCamera = await left.evaluate((v) => v.currentTime);
|
||||
await bench.getByRole("button", { name: "放大左相机", exact: true }).click();
|
||||
await page.waitForFunction(
|
||||
(t) => document.querySelector('video[data-camera="left"]').currentTime > t + 0.3,
|
||||
beforeSameCamera,
|
||||
);
|
||||
// One transient read failure recovers automatically and retains play intent.
|
||||
fail = true;
|
||||
await left.evaluate((v) => v.load());
|
||||
await page.waitForFunction(() => !!document.querySelector('video[data-camera="left"]').error);
|
||||
fail = false;
|
||||
await settle();
|
||||
await page.waitForFunction(() => !document.querySelector('video[data-camera="left"]').paused);
|
||||
failedRequests = 0;
|
||||
await bench.getByRole("button", { name: "放大右相机", exact: true }).click();
|
||||
await settle();
|
||||
assert.equal(await left.getAttribute("src"), null);
|
||||
assert((await right.evaluate((v) => v.currentTime)) > 0.8, "camera switch preserves logical playback position");
|
||||
await bench.getByRole("button", { name: "恢复双画面", exact: true }).click();
|
||||
await settle();
|
||||
await bench.getByRole("button", { name: "暂停", exact: true }).click();
|
||||
await bench.getByRole("button", { name: "放大左相机", exact: true }).click();
|
||||
await settle();
|
||||
const held = await left.evaluate((v) => v.currentTime);
|
||||
let input;
|
||||
if (mode === "annotation") {
|
||||
await bench.getByTestId("annotation-description-block").first().click();
|
||||
input = bench.getByRole("textbox", { name: "动作描述", exact: true });
|
||||
await input.fill("Unsubmitted draft must survive video recovery");
|
||||
}
|
||||
const beforeFailure = await left.evaluate((v) => v.currentTime);
|
||||
fail = true;
|
||||
await left.evaluate((v) => v.load());
|
||||
await page.waitForTimeout(4500);
|
||||
assert(failedRequests <= 3, `automatic retries must be bounded: ${failedRequests}`);
|
||||
assert(await left.evaluate((v) => !!v.error), "persistent failure remains visible in the media element");
|
||||
assert(await bench.getByRole("alert").count(), "do not clear the failure before media recovers");
|
||||
fail = false;
|
||||
await bench.getByRole("button", { name: "下一帧", exact: true }).click();
|
||||
await settle();
|
||||
assert(
|
||||
requests.some((r) => r.retry),
|
||||
"retry returns to original authorized URL for a fresh signature",
|
||||
);
|
||||
assert(
|
||||
Math.abs((await left.evaluate((v) => v.currentTime)) - beforeFailure) < 0.15,
|
||||
"recovery preserves the frame clock",
|
||||
);
|
||||
assert.equal(await left.evaluate((v) => v.paused), true, "paused users are not auto-played");
|
||||
if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery");
|
||||
// Explicit retry also works after the bounded automatic attempts are exhausted.
|
||||
fail = true;
|
||||
failedRequests = 0;
|
||||
await left.evaluate((v) => v.load());
|
||||
await page.waitForTimeout(4500);
|
||||
fail = false;
|
||||
await bench.getByRole("button", { name: "重新加载视频", exact: true }).click();
|
||||
await settle();
|
||||
assert.equal(await bench.getByRole("alert").count(), 0);
|
||||
if (input) assert.equal(await input.inputValue(), "Unsubmitted draft must survive video recovery");
|
||||
assert.deepEqual(errors, []);
|
||||
console.log(JSON.stringify({ mode, passed: true, held, requests: requests.length }));
|
||||
await page.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
@ -67,8 +67,8 @@ test("both playhead sliders use latest-only pointer previews and exact final com
|
||||
});
|
||||
|
||||
test("hidden sources detach and right-only playback maps back to the canonical frame clock", () => {
|
||||
assert.match(workbench, /const effectiveShowLeft = focusedLayout \|\| showLeft/);
|
||||
assert.match(workbench, /const effectiveShowRight = focusedLayout \? Boolean\(rightVideoSource\) : showRight/);
|
||||
// Real focused-camera resource lifetime is exercised by
|
||||
// tests/browser/video-playback-recovery.mjs (hidden camera issues no request).
|
||||
assert.match(workbench, /src=\{effectiveShowLeft \? source : undefined\}/);
|
||||
assert.match(workbench, /src=\{effectiveShowRight \? rightSource : undefined\}/);
|
||||
assert.match(workbench, /mapCameraBoundaryTime\(rightMediaBoundaries, frameBoundaries, time\)/);
|
||||
|
||||
40
tests/frontend/invalid-slice-consolidation.test.mjs
Normal file
40
tests/frontend/invalid-slice-consolidation.test.mjs
Normal file
@ -0,0 +1,40 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { invalidSliceConsolidationCandidates as candidates } from "../../src/lib/invalid-slice-consolidation.js";
|
||||
const slices = ["a", "b", "c"].map((id, i) => ({ id, name: "盘点", start: i, end: i + 1 }));
|
||||
const actions = [{ id: "invalid", start: 1, end: 2, isInvalid: true, description: "无效片段" }];
|
||||
test("identify invalid-only outer partition without modifying slices or action JSON", () => {
|
||||
const before = JSON.stringify({ slices, actions });
|
||||
assert.deepEqual(candidates(slices, actions), [
|
||||
{ id: "b", name: "盘点", start: 0, end: 3, invalidStart: 1, invalidEnd: 2 },
|
||||
]);
|
||||
assert.equal(JSON.stringify({ slices, actions }), before);
|
||||
});
|
||||
test("never propose discarded, different-name, gapped or partially invalid partitions", () => {
|
||||
assert.deepEqual(
|
||||
candidates(
|
||||
slices.map((s) => (s.id === "b" ? { ...s, deliveryDisposition: "discard" } : s)),
|
||||
actions,
|
||||
),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(
|
||||
candidates(
|
||||
slices.map((s) => (s.id === "c" ? { ...s, name: "清洁" } : s)),
|
||||
actions,
|
||||
),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(
|
||||
candidates(
|
||||
slices.map((s) => (s.id === "c" ? { ...s, start: 2.1 } : s)),
|
||||
actions,
|
||||
),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(candidates(slices, [{ ...actions[0], end: 1.5 }]), []);
|
||||
assert.deepEqual(candidates(slices, [{ ...actions[0], isInvalid: false }]), []);
|
||||
});
|
||||
test("invalid inside a single business slice stays inside it", () => {
|
||||
assert.deepEqual(candidates([{ id: "a", name: "盘点", start: 0, end: 3 }], actions), []);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user