313 lines
15 KiB
Go
313 lines
15 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
func seedFailedSliceRetry(t *testing.T, f combinedDraftFixture) domain.HumanItemRecord {
|
|
t.Helper()
|
|
h := f.item
|
|
h.Subject.Payload["sliceWorkflow"] = map[string]any{
|
|
"status": "failed", "phase": "materializing", "requestId": "legacy-interrupted",
|
|
"revision": 1, "errorCode": "slice_processing_interrupted", "error": "retained failure",
|
|
"startedAt": "2026-09-23T13:25:10Z", "updatedAt": "2026-09-23T13:35:01Z",
|
|
}
|
|
h.Subject.Payload["annotationWorkspace"] = map[string]any{
|
|
"sliceSegments": []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 0, End: 2}, {ID: "b", Name: "测试切片", Start: 2, End: 4}},
|
|
"operations": []domain.SliceOperation{},
|
|
"segments": []domain.AnnotationSegment{{ID: "retained-action", Start: 0, End: 2, Status: "pending"}},
|
|
"statuses": map[string]domain.SliceAnnotationStatus{"a": domain.SliceAnnotationPending, "b": domain.SliceAnnotationPending},
|
|
"userId": f.annotator.ID, "savedAt": "2026-09-23T13:25:10Z",
|
|
}
|
|
if e := f.store.UpsertHumanItem(f.ctx, h); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
return h
|
|
}
|
|
|
|
func TestFailedSliceRetryUsesRetainedWorkspaceAndDurableRecovery(t *testing.T) {
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
f, root := annotationProcessorFixture(t)
|
|
seedFailedSliceRetry(t, f)
|
|
h, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
|
before := annotationWorkspacePayload(h)
|
|
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
|
|
map[string]any{"version": h.Version, "retryFailed": true})
|
|
if r.Code != http.StatusAccepted {
|
|
t.Fatalf("retained retry: %d %s", r.Code, r.Body.String())
|
|
}
|
|
h, _ = f.store.GetHumanItem(f.ctx, h.ID)
|
|
if h.ProcessingJob == nil || h.ProcessingJob.Kind != "slice" || h.ProcessingJob.Status != "queued" || h.ProcessingJob.ID == "legacy-interrupted" {
|
|
t.Fatal("retry was not atomically queued as a new durable job")
|
|
}
|
|
if !reflect.DeepEqual(annotationWorkspacePayload(h), before) || h.AssignedUserID != f.annotator.ID || h.CurrentStageID != domain.FanyuV58StageAnnotationCombined {
|
|
t.Fatal("accepting a retry changed retained content or task ownership")
|
|
}
|
|
flow := mapValue(h.Subject.Payload["sliceWorkflow"])
|
|
if stringValue(mapValue(flow["retryOf"])["errorCode"]) != "slice_processing_interrupted" {
|
|
t.Fatal("original failure evidence was lost")
|
|
}
|
|
duplicate := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
|
|
map[string]any{"version": h.Version, "retryFailed": true})
|
|
if duplicate.Code != http.StatusConflict {
|
|
t.Fatalf("duplicate retry accepted: %d %s", duplicate.Code, duplicate.Body.String())
|
|
}
|
|
now := time.Now().UTC()
|
|
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "retry-old", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
|
|
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "retry-new", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
|
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
|
|
t.Fatal("retry job was not reclaimable after interruption")
|
|
}
|
|
if ok, _ := f.store.UpdateHumanItemVersioned(f.ctx, *old, old.Version); ok {
|
|
t.Fatal("stale worker could publish")
|
|
}
|
|
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
|
|
s.executeAnnotationJob(context.Background(), *claimed)
|
|
h, _ = f.store.GetHumanItem(f.ctx, h.ID)
|
|
if h.ProcessingJob.Status != "completed" || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["phase"]) != "annotate" {
|
|
t.Fatalf("retry did not complete: %+v", h.ProcessingJob)
|
|
}
|
|
workspace := annotationWorkspacePayload(h)
|
|
if !reflect.DeepEqual(decodeMaterializationValue[domain.AnnotationSegment](workspace["segments"]), decodeMaterializationValue[domain.AnnotationSegment](before["segments"])) {
|
|
t.Fatal("successful retry erased retained actions")
|
|
}
|
|
if stringValue(mapValue(mapValue(h.Subject.Payload["sliceWorkflow"])["retryOf"])["requestId"]) != "legacy-interrupted" {
|
|
t.Fatal("completion erased original failure evidence")
|
|
}
|
|
}
|
|
|
|
func TestLegacyGenerationRecoveryPreservesAnnotationsAcrossRestart(t *testing.T) {
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
f, root := annotationProcessorFixture(t)
|
|
h := seedFailedSliceRetry(t, f)
|
|
delete(h.Subject.Payload, "sliceWorkflow")
|
|
delete(h.Subject.Payload, "materialization")
|
|
workspace := annotationWorkspacePayload(&h)
|
|
h.Subject.Payload["annotationWorkspace"] = workspace
|
|
draft := map[string]any{}
|
|
for key, value := range workspace {
|
|
draft[key] = value
|
|
}
|
|
draft["segments"] = []domain.AnnotationSegment{{ID: "retained-action", Start: 0, End: 2, Status: "pending"}, {ID: "human-description", Start: 2, End: 4, Status: "described", Description: "人工确认的分类内容"}}
|
|
draft["savedAt"] = nowISO()
|
|
draft["savedVersion"] = h.Version
|
|
h.Subject.Payload["annotationDraft"] = draft
|
|
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
saved, err := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h = *saved
|
|
workspace = annotationWorkspacePayload(&h)
|
|
draft = mapValue(h.Subject.Payload["annotationDraft"])
|
|
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
|
|
state, err := s.annotationEditorStateForTask(f.ctx, f.task, &h)
|
|
if err != nil || state.Phase != "slice" || !state.CanRecoverGeneration || state.CanConfirmDescriptions {
|
|
t.Fatalf("state=%+v err=%v", state, err)
|
|
}
|
|
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true, "retryWorkspace": map[string]any{"segments": []any{}}})
|
|
if r.Code != http.StatusAccepted {
|
|
t.Fatalf("recover: %d %s", r.Code, r.Body.String())
|
|
}
|
|
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if !reflect.DeepEqual(annotationWorkspacePayload(queued), workspace) {
|
|
t.Fatal("queueing recovery erased annotations")
|
|
}
|
|
if !reflect.DeepEqual(mapValue(queued.Subject.Payload["annotationDraft"]), draft) {
|
|
t.Fatal("queueing recovery erased the newer autosaved draft")
|
|
}
|
|
if duplicate := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": queued.Version, "recoverExisting": true}); duplicate.Code != 409 {
|
|
t.Fatal("duplicate recovery accepted")
|
|
}
|
|
now := time.Now().UTC()
|
|
old, _ := f.store.ClaimAnnotationProcessing(f.ctx, "recovery-old", now.Format(time.RFC3339Nano), now.Add(-time.Second).Format(time.RFC3339Nano))
|
|
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "recovery-new", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
|
if old == nil || claimed == nil || claimed.ProcessingJob.Attempt != 2 {
|
|
t.Fatal("recovery did not survive worker restart")
|
|
}
|
|
// Exercise a real processor rejection before retrying: both snapshots must
|
|
// survive failure, and retry must not switch back to the older baseline.
|
|
claimed.ProcessingJob.SourceFingerprint = "synthetic-source-changed"
|
|
s.executeAnnotationJob(context.Background(), *claimed)
|
|
failed, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if failed.ProcessingJob.Status != "failed" || !reflect.DeepEqual(annotationWorkspacePayload(failed), workspace) || !reflect.DeepEqual(mapValue(failed.Subject.Payload["annotationDraft"]), draft) {
|
|
t.Fatal("failed recovery changed retained snapshots")
|
|
}
|
|
r = f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": failed.Version, "retryFailed": true})
|
|
if r.Code != http.StatusAccepted {
|
|
t.Fatalf("retry recovery: %d %s", r.Code, r.Body.String())
|
|
}
|
|
claimed, _ = f.store.ClaimAnnotationProcessing(f.ctx, "recovery-retry", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
|
if claimed == nil {
|
|
t.Fatal("recovery retry was not queued")
|
|
}
|
|
s.executeAnnotationJob(context.Background(), *claimed)
|
|
completed, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if completed.ProcessingJob.Status != "completed" {
|
|
t.Fatalf("recovery failed: %+v", completed.ProcessingJob)
|
|
}
|
|
if !reflect.DeepEqual(decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(completed)["segments"]), decodeMaterializationValue[domain.AnnotationSegment](draft["segments"])) {
|
|
t.Fatal("generation lost human descriptions or pending actions")
|
|
}
|
|
state, err = s.annotationEditorStateForTask(f.ctx, f.task, completed)
|
|
if err != nil || state.Phase != "annotate" || !state.CanConfirmDescriptions || state.CanRecoverGeneration {
|
|
t.Fatalf("completed state=%+v err=%v", state, err)
|
|
}
|
|
}
|
|
|
|
func TestLegacyGenerationRecoveryRecognizesDraftBeyondEmptyBaseline(t *testing.T) {
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
f, _ := annotationProcessorFixture(t)
|
|
h := seedFailedSliceRetry(t, f)
|
|
delete(h.Subject.Payload, "sliceWorkflow")
|
|
workspace := annotationWorkspacePayload(&h)
|
|
draft := mapValue(workspace)
|
|
draft["savedAt"] = nowISO()
|
|
workspace["segments"] = []domain.AnnotationSegment{}
|
|
h.Subject.Payload["annotationWorkspace"] = workspace
|
|
h.Subject.Payload["annotationDraft"] = draft
|
|
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state, err := (&server{store: f.store}).annotationEditorStateForTask(f.ctx, f.task, &h)
|
|
if err != nil || !state.CanRecoverGeneration || state.CanConfirmDescriptions {
|
|
t.Fatalf("draft-only work was not recoverable: %+v %v", state, err)
|
|
}
|
|
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true})
|
|
if r.Code != http.StatusAccepted {
|
|
t.Fatalf("recover draft-only work: %d %s", r.Code, r.Body.String())
|
|
}
|
|
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if len(decodeMaterializationValue[domain.AnnotationSegment](mapValue(queued.ProcessingJob.Input["retryWorkspace"])["segments"])) != 1 {
|
|
t.Fatal("empty baseline replaced the newer draft in the durable job")
|
|
}
|
|
}
|
|
|
|
func TestLegacyGenerationRecoveryRejectsAmbiguousDraftWithoutMutation(t *testing.T) {
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
for _, name := range []string{"wrong-owner", "future-version", "missing-time", "malformed-actions"} {
|
|
t.Run(name, func(t *testing.T) {
|
|
f, _ := annotationProcessorFixture(t)
|
|
h := seedFailedSliceRetry(t, f)
|
|
delete(h.Subject.Payload, "sliceWorkflow")
|
|
draft := map[string]any{}
|
|
for key, value := range annotationWorkspacePayload(&h) {
|
|
draft[key] = value
|
|
}
|
|
draft["savedAt"] = nowISO()
|
|
switch name {
|
|
case "wrong-owner":
|
|
draft["userId"] = f.reviewer.ID
|
|
case "future-version":
|
|
draft["savedVersion"] = h.Version + 1
|
|
case "missing-time":
|
|
delete(draft, "savedAt")
|
|
case "malformed-actions":
|
|
draft["segments"] = "not-a-list"
|
|
}
|
|
h.Subject.Payload["annotationDraft"] = draft
|
|
if err := f.store.UpsertHumanItem(f.ctx, h); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
r := f.request("POST", "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator, map[string]any{"version": h.Version, "recoverExisting": true})
|
|
if r.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("ambiguous draft accepted: %d %s", r.Code, r.Body.String())
|
|
}
|
|
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if !reflect.DeepEqual(before, after) {
|
|
t.Fatal("rejected recovery changed saved snapshots")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFailedSliceRetryRejectsUnsafeSnapshotsWithoutMutation(t *testing.T) {
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
f, _ := annotationProcessorFixture(t)
|
|
for _, name := range []string{"missing", "malformed", "stale", "active", "not-failed", "wrong-owner", "bad-name", "bad-boundaries"} {
|
|
t.Run(name, func(t *testing.T) {
|
|
h := seedFailedSliceRetry(t, f)
|
|
version := h.Version
|
|
switch name {
|
|
case "missing":
|
|
delete(h.Subject.Payload, "annotationWorkspace")
|
|
case "malformed":
|
|
h.Subject.Payload["annotationWorkspace"].(map[string]any)["segments"] = "not-a-list"
|
|
case "stale":
|
|
version--
|
|
case "active":
|
|
h.ProcessingJob = &domain.AnnotationProcessingJob{ID: "live", Kind: "slice", Status: "running"}
|
|
case "not-failed":
|
|
h.Subject.Payload["sliceWorkflow"].(map[string]any)["status"] = "completed"
|
|
case "wrong-owner":
|
|
h.AssignedUserID = f.reviewer.ID
|
|
case "bad-name":
|
|
h.Subject.Payload["annotationWorkspace"].(map[string]any)["sliceSegments"] = []domain.SliceSegment{{ID: "a", Name: "not-in-dictionary", Start: 0, End: 4}}
|
|
case "bad-boundaries":
|
|
h.Subject.Payload["annotationWorkspace"].(map[string]any)["sliceSegments"] = []domain.SliceSegment{{ID: "a", Name: "测试切片", Start: 1, End: 4}}
|
|
}
|
|
f.store.UpsertHumanItem(f.ctx, h)
|
|
before, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
|
|
map[string]any{"version": version, "retryFailed": true})
|
|
if r.Code < 400 || r.Code >= 500 {
|
|
t.Fatalf("unsafe snapshot accepted or crashed: %d %s", r.Code, r.Body.String())
|
|
}
|
|
after, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
if !reflect.DeepEqual(before, after) {
|
|
t.Fatalf("rejected retry mutated stored state: %s", name)
|
|
}
|
|
if name == "stale" && !strings.Contains(r.Body.String(), "version_conflict") {
|
|
t.Fatal("stale retry did not return version conflict")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter(t *testing.T) {
|
|
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
|
f, root := annotationProcessorFixture(t)
|
|
h := seedFailedSliceRetry(t, f)
|
|
h.Subject.Payload["annotationDraft"] = h.Subject.Payload["annotationWorkspace"]
|
|
delete(h.Subject.Payload, "annotationWorkspace")
|
|
f.store.UpsertHumanItem(f.ctx, h)
|
|
r := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/slice-confirmations", f.annotator,
|
|
map[string]any{"version": h.Version, "reconfirmSaved": true, "retryWorkspace": map[string]any{"segments": []any{}}})
|
|
if r.Code != 202 {
|
|
t.Fatal(r.Code, r.Body.String())
|
|
}
|
|
queued, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
r = f.request(http.MethodPut, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/drafts", f.annotator, map[string]any{"version": queued.Version})
|
|
if r.Code != 409 || !strings.Contains(r.Body.String(), "slice_processing") {
|
|
t.Fatalf("draft write during retry: %d %s", r.Code, r.Body.String())
|
|
}
|
|
now := time.Now().UTC()
|
|
claimed, _ := f.store.ClaimAnnotationProcessing(f.ctx, "snapshot-worker", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
|
// Simulate an old API replica that was already in a draft write during rollout.
|
|
changed, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
draft := mapValue(changed.Subject.Payload["annotationDraft"])
|
|
draft["segments"] = []domain.AnnotationSegment{}
|
|
changed.Subject.Payload["annotationDraft"] = draft
|
|
changed.Version++
|
|
f.store.UpsertHumanItem(f.ctx, *changed)
|
|
s := &server{store: f.store, workspaceMountPath: root, externalAnnotationProcessor: true}
|
|
s.executeAnnotationJob(context.Background(), *claimed)
|
|
saved, _ := f.store.GetHumanItem(f.ctx, h.ID)
|
|
actions := decodeMaterializationValue[domain.AnnotationSegment](annotationWorkspacePayload(saved)["segments"])
|
|
retained := mapValue(saved.ProcessingJob.Input["retryWorkspace"])
|
|
retainedActions := decodeMaterializationValue[domain.AnnotationSegment](retained["segments"])
|
|
if saved.ProcessingJob.Status != "failed" || len(actions) != 0 || len(retainedActions) != 1 || retainedActions[0].ID != "retained-action" {
|
|
t.Fatal("late edit or original job snapshot was lost", saved.ProcessingJob, actions)
|
|
}
|
|
}
|