droboticflow/internal/api/slice_retry_test.go
d-robotics 63512da71f
Some checks failed
web-ci / checks (push) Waiting to run
web-ci / checks (pull_request) Has been cancelled
🐛 Preserve confirmed annotation frames and enforce v4 time evidence
2026-10-22 12:54:08 +08:00

313 lines
15 KiB
Go

package api
import (
"context"
"net/http"
"reflect"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
)
func seedFailedSliceRetry(t *testing.T, f combinedDraftFixture) domain.HumanItemRecord {
t.Helper()
h := f.item
h.Subject.Payload["sliceWorkflow"] = map[string]any{
"status": "failed", "phase": "materializing", "requestId": "legacy-interrupted",
"revision": 1, "errorCode": "slice_processing_interrupted", "error": "retained failure",
"startedAt": "2026-09-23T13:25:10Z", "updatedAt": "2026-09-23T13:35:01Z",
}
h.Subject.Payload["annotationWorkspace"] = map[string]any{
"sliceSegments": []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 2}, {ID: "b", Name: "测试切片", Start: 2, End: 4}},
"operations": []domain.SliceOperation{},
"segments": []domain.AnnotationSegment{{ID: "retained-action", Start: 0, End: 2, Status: "pending"}},
"statuses": map[string]domain.SliceAnnotationStatus{"a": domain.SliceAnnotationPending, "b": domain.SliceAnnotationPending},
"userId": f.annotator.ID, "savedAt": "2026-09-23T13:25:10Z",
}
if e := f.store.UpsertHumanItem(f.ctx, h); e != nil {
t.Fatal(e)
}
return h
}
func TestFailedSliceRetryUsesRetainedWorkspaceAndDurableRecovery(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
seedFailedSliceRetry(t, f)
h, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
before := annotationWorkspacePayload(h)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
map[string]any{"version": h.Version, "retryFailed": true})
if r.Code != http.StatusAccepted {
t.Fatalf("retained retry: %d %s", r.Code, r.Body.String())
}
h, _ = f.store.GetHumanItem(f.ctx, h.ID)
if h.ProcessingJob == nil || h.ProcessingJob.Kind != "slice" || h.ProcessingJob.Status != "queued" || h.ProcessingJob.ID == "legacy-interrupted" {
t.Fatal("retry was not atomically queued as a new durable job")
}
if !reflect.DeepEqual(annotationWorkspacePayload(h), before) || h.AssignedUserID != f.annotator.ID || h.CurrentStageID != domain.FanyuV58StageAnnotationCombined {
t.Fatal("accepting a retry changed retained content or task ownership")
}
flow := mapValue(h.Subject.Payload["sliceWorkflow"])
if stringValue(mapValue(flow["retryOf"])["errorCode"]) != "slice_processing_interrupted" {
t.Fatal("original failure evidence was lost")
}
duplicate := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
map[string]any{"version": h.Version, "retryFailed": true})
if duplicate.Code != http.StatusConflict {
t.Fatalf("duplicate retry accepted: %d %s", duplicate.Code, duplicate.Body.String())
}
now := time.Now().UTC()
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "retry-old", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "retry-new", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
t.Fatal("retry job was not reclaimable after interruption")
}
if ok, _ := f.store.UpdateHumanItemVersioned(f.ctx, *old, old.Version); ok {
t.Fatal("stale worker could publish")
}
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
s.executeAnnotationJob(context.Background(), *claimed)
h, _ = f.store.GetHumanItem(f.ctx, h.ID)
if h.ProcessingJob.Status != "completed" || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["phase"]) != "annotate" {
t.Fatalf("retry did not complete: %+v", h.ProcessingJob)
}
workspace := annotationWorkspacePayload(h)
if !reflect.DeepEqual(decodeMaterializationValue[domain.AnnotationSegment](workspace["segments"]), decodeMaterializationValue[domain.AnnotationSegment](before["segments"])) {
t.Fatal("successful retry erased retained actions")
}
if stringValue(mapValue(mapValue(h.Subject.Payload["sliceWorkflow"])["retryOf"])["requestId"]) != "legacy-interrupted" {
t.Fatal("completion erased original failure evidence")
}
}
func TestLegacyGenerationRecoveryPreservesAnnotationsAcrossRestart(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
delete(h.Subject.Payload, "sliceWorkflow")
delete(h.Subject.Payload, "materialization")
workspace := annotationWorkspacePayload(&h)
h.Subject.Payload["annotationWorkspace"] = workspace
draft := map[string]any{}
for key, value := range workspace {
draft[key] = value
}
draft["segments"] = []domain.AnnotationSegment{{ID: "retained-action", Start: 0, End: 2, Status: "pending"}, {ID: "human-description", Start: 2, End: 4, Status: "described", Description: "人工确认的分类内容"}}
draft["savedAt"] = nowISO()
draft["savedVersion"] = h.Version
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
saved, err := f.store.GetHumanItem(f.ctx, h.ID)
if err != nil {
t.Fatal(err)
}
h = *saved
workspace = annotationWorkspacePayload(&h)
draft = mapValue(h.Subject.Payload["annotationDraft"])
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
state, err := s.annotationEditorStateForTask(f.ctx, f.task, &h)
if err != nil || state.Phase != "slice" || !state.CanRecoverGeneration || state.CanConfirmDescriptions {
t.Fatalf("state=%+v err=%v", state, err)
}
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true, "retryWorkspace": map[string]any{"segments": []any{}}})
if r.Code != http.StatusAccepted {
t.Fatalf("recover: %d %s", r.Code, r.Body.String())
}
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(annotationWorkspacePayload(queued), workspace) {
t.Fatal("queueing recovery erased annotations")
}
if !reflect.DeepEqual(mapValue(queued.Subject.Payload["annotationDraft"]), draft) {
t.Fatal("queueing recovery erased the newer autosaved draft")
}
if duplicate := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": queued.Version, "recoverExisting": true}); duplicate.Code != 409 {
t.Fatal("duplicate recovery accepted")
}
now := time.Now().UTC()
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "recovery-old", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "recovery-new", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
t.Fatal("recovery did not survive worker restart")
}
// Exercise a real processor rejection before retrying: both snapshots must
// survive failure, and retry must not switch back to the older baseline.
claimed.ProcessingJob.SourceFingerprint = "synthetic-source-changed"
s.executeAnnotationJob(context.Background(), *claimed)
failed, _ := f.store.GetHumanItem(f.ctx, h.ID)
if failed.ProcessingJob.Status != "failed" || !reflect.DeepEqual(annotationWorkspacePayload(failed), workspace) || !reflect.DeepEqual(mapValue(failed.Subject.Payload["annotationDraft"]), draft) {
t.Fatal("failed recovery changed retained snapshots")
}
r = f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": failed.Version, "retryFailed": true})
if r.Code != http.StatusAccepted {
t.Fatalf("retry recovery: %d %s", r.Code, r.Body.String())
}
claimed, _ = f.store.ClaimAnnotationProcessing(f.ctx, "recovery-retry", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
if claimed == nil {
t.Fatal("recovery retry was not queued")
}
s.executeAnnotationJob(context.Background(), *claimed)
completed, _ := f.store.GetHumanItem(f.ctx, h.ID)
if completed.ProcessingJob.Status != "completed" {
t.Fatalf("recovery failed: %+v", completed.ProcessingJob)
}
if !reflect.DeepEqual(decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(completed)["segments"]), decodeMaterializationValue[domain.AnnotationSegment](draft["segments"])) {
t.Fatal("generation lost human descriptions or pending actions")
}
state, err = s.annotationEditorStateForTask(f.ctx, f.task, completed)
if err != nil || state.Phase != "annotate" || !state.CanConfirmDescriptions || state.CanRecoverGeneration {
t.Fatalf("completed state=%+v err=%v", state, err)
}
}
func TestLegacyGenerationRecoveryRecognizesDraftBeyondEmptyBaseline(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, _ := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
delete(h.Subject.Payload, "sliceWorkflow")
workspace := annotationWorkspacePayload(&h)
draft := mapValue(workspace)
draft["savedAt"] = nowISO()
workspace["segments"] = []domain.AnnotationSegment{}
h.Subject.Payload["annotationWorkspace"] = workspace
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
state, err := (&server{store: f.store}).annotationEditorStateForTask(f.ctx, f.task, &h)
if err != nil || !state.CanRecoverGeneration || state.CanConfirmDescriptions {
t.Fatalf("draft-only work was not recoverable: %+v %v", state, err)
}
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true})
if r.Code != http.StatusAccepted {
t.Fatalf("recover draft-only work: %d %s", r.Code, r.Body.String())
}
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
if len(decodeMaterializationValue[domain.AnnotationSegment](mapValue(queued.ProcessingJob.Input["retryWorkspace"])["segments"])) != 1 {
t.Fatal("empty baseline replaced the newer draft in the durable job")
}
}
func TestLegacyGenerationRecoveryRejectsAmbiguousDraftWithoutMutation(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
for _, name := range []string{"wrong-owner", "future-version", "missing-time", "malformed-actions"} {
t.Run(name, func(t *testing.T) {
f, _ := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
delete(h.Subject.Payload, "sliceWorkflow")
draft := map[string]any{}
for key, value := range annotationWorkspacePayload(&h) {
draft[key] = value
}
draft["savedAt"] = nowISO()
switch name {
case "wrong-owner":
draft["userId"] = f.reviewer.ID
case "future-version":
draft["savedVersion"] = h.Version + 1
case "missing-time":
delete(draft, "savedAt")
case "malformed-actions":
draft["segments"] = "not-a-list"
}
h.Subject.Payload["annotationDraft"] = draft
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
t.Fatal(err)
}
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true})
if r.Code != http.StatusUnprocessableEntity {
t.Fatalf("ambiguous draft accepted: %d %s", r.Code, r.Body.String())
}
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(before, after) {
t.Fatal("rejected recovery changed saved snapshots")
}
})
}
}
func TestFailedSliceRetryRejectsUnsafeSnapshotsWithoutMutation(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, _ := annotationProcessorFixture(t)
for _, name := range []string{"missing", "malformed", "stale", "active", "not-failed", "wrong-owner", "bad-name", "bad-boundaries"} {
t.Run(name, func(t *testing.T) {
h := seedFailedSliceRetry(t, f)
version := h.Version
switch name {
case "missing":
delete(h.Subject.Payload, "annotationWorkspace")
case "malformed":
h.Subject.Payload["annotationWorkspace"].(map[string]any)["segments"] = "not-a-list"
case "stale":
version--
case "active":
h.ProcessingJob = &domain.AnnotationProcessingJob{ID: "live", Kind: "slice", Status: "running"}
case "not-failed":
h.Subject.Payload["sliceWorkflow"].(map[string]any)["status"] = "completed"
case "wrong-owner":
h.AssignedUserID = f.reviewer.ID
case "bad-name":
h.Subject.Payload["annotationWorkspace"].(map[string]any)["sliceSegments"] = []domain.SliceSegment{{ID: "a", Name: "not-in-dictionary", Start: 0, End: 4}}
case "bad-boundaries":
h.Subject.Payload["annotationWorkspace"].(map[string]any)["sliceSegments"] = []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 1, End: 4}}
}
f.store.UpsertHumanItem(f.ctx, h)
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
map[string]any{"version": version, "retryFailed": true})
if r.Code < 400 || r.Code >= 500 {
t.Fatalf("unsafe snapshot accepted or crashed: %d %s", r.Code, r.Body.String())
}
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
if !reflect.DeepEqual(before, after) {
t.Fatalf("rejected retry mutated stored state: %s", name)
}
if name == "stale" && !strings.Contains(r.Body.String(), "version_conflict") {
t.Fatal("stale retry did not return version conflict")
}
})
}
}
func TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter(t *testing.T) {
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
f, root := annotationProcessorFixture(t)
h := seedFailedSliceRetry(t, f)
h.Subject.Payload["annotationDraft"] = h.Subject.Payload["annotationWorkspace"]
delete(h.Subject.Payload, "annotationWorkspace")
f.store.UpsertHumanItem(f.ctx, h)
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
map[string]any{"version": h.Version, "reconfirmSaved": true, "retryWorkspace": map[string]any{"segments": []any{}}})
if r.Code != 202 {
t.Fatal(r.Code, r.Body.String())
}
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
r = f.request(http.MethodPut, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/drafts", f.annotator, map[string]any{"version": queued.Version})
if r.Code != 409 || !strings.Contains(r.Body.String(), "slice_processing") {
t.Fatalf("draft write during retry: %d %s", r.Code, r.Body.String())
}
now := time.Now().UTC()
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "snapshot-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
// Simulate an old API replica that was already in a draft write during rollout.
changed, _ := f.store.GetHumanItem(f.ctx, h.ID)
draft := mapValue(changed.Subject.Payload["annotationDraft"])
draft["segments"] = []domain.AnnotationSegment{}
changed.Subject.Payload["annotationDraft"] = draft
changed.Version++
f.store.UpsertHumanItem(f.ctx, *changed)
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
s.executeAnnotationJob(context.Background(), *claimed)
saved, _ := f.store.GetHumanItem(f.ctx, h.ID)
actions := decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(saved)["segments"])
retained := mapValue(saved.ProcessingJob.Input["retryWorkspace"])
retainedActions := decodeMaterializationValue[domain.AnnotationSegment](retained["segments"])
if saved.ProcessingJob.Status != "failed" || len(actions) != 0 || len(retainedActions) != 1 || retainedActions[0].ID != "retained-action" {
t.Fatal("late edit or original job snapshot was lost", saved.ProcessingJob, actions)
}
}