droboticflow/internal/worker/capture_cutover_test.go
2026-11-08 10:00:31 -08:00

590 lines
26 KiB
Go

package worker
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"droboticflow/internal/api"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
type captureCutoverFixture struct {
t *testing.T
ctx context.Context
db *store.MemoryStore
runtime *Runtime
old, next domain.WorkflowRecord
objects []captureObject
beforeList func()
}
func newCaptureCutoverFixture(t *testing.T, names ...string) *captureCutoverFixture {
t.Helper()
f := &captureCutoverFixture{t: t, ctx: context.Background(), db: store.NewMemoryStore()}
for _, name := range names {
for _, object := range append(calibrationObjects(), partObjects("part0001")...) {
object.Key = strings.ReplaceAll(object.Key, layoutCapture, name)
f.objects = append(f.objects, object)
}
f.objects = append(f.objects, captureObject{Key: "incoming/" + name + "/meta_" + name + ".json", Size: 10, ETag: "metadata"})
}
bos := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if f.beforeList != nil {
f.beforeList()
}
contents := []map[string]any{}
for _, object := range f.objects {
contents = append(contents, map[string]any{"key": object.Key, "eTag": object.ETag, "size": object.Size})
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"contents": contents, "isTruncated": false})
}))
t.Cleanup(bos.Close)
project, err := f.db.CreateProject(f.ctx, domain.CreateProjectInput{OrganizationID: domain.DefaultOrganizationID, Name: "cutover"})
if err != nil {
t.Fatal(err)
}
if err := f.db.UpsertStorageConnection(f.ctx, domain.StorageConnectionRecord{ID: "cutover-storage", OrganizationID: project.OrganizationID, Provider: domain.StorageProviderBOS, Bucket: "synthetic", Endpoint: bos.URL, AccessKeyID: "synthetic", SecretAccessKey: "synthetic"}); err != nil {
t.Fatal(err)
}
config := &domain.CaptureAutomationConfig{Enabled: true, ConnectionID: "cutover-storage", Bucket: "synthetic", Prefix: "incoming", WorkflowInputID: "sourceDirectory", DeliveryPrefix: "deliveries", DeliveryProjectDirectory: "cutover__project", StabilityWindowSeconds: 1}
f.old = domain.WorkflowRecord{ID: "cutover-old", OrganizationID: project.OrganizationID, ProjectID: project.ID, Type: domain.WorkflowTypeStreaming, ExecutionScope: domain.WorkflowExecutionScopeProduction, DefinitionState: domain.WorkflowDefinitionStateReleased, DefinitionFamilyID: "shared-template-family", DefinitionVersion: 1, CaptureAutomation: config, WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory", Name: "sourceDirectory", Type: domain.PortTypeFile, Container: domain.PortContainerDirectory, PayloadMode: domain.ArtifactTransferModeRef, Required: true}}, Nodes: []domain.WorkflowNode{{ID: "package", Data: domain.WorkflowNodeData{OperatorID: "operator-builtin-package", Image: "registry.example/package:legacy", OperatorImageID: "registry.example/package@sha256:" + strings.Repeat("a", 64), Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: "false"}, {Key: "PIN_OPERATOR_IMAGE", Value: "true"}}}}}}
f.next = f.old
f.next.ID, f.next.Hidden, f.next.DefinitionVersion, f.next.SupersedesWorkflowID = "cutover-new", true, 2, f.old.ID
f.next.Nodes = []domain.WorkflowNode{{ID: "package", Data: domain.WorkflowNodeData{OperatorID: "operator-builtin-package", Image: "registry.example/package:v4", OperatorImageID: "registry.example/package@sha256:" + strings.Repeat("b", 64), Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: "true"}, {Key: "PIN_OPERATOR_IMAGE", Value: "true"}}}}}
for _, workflow := range []domain.WorkflowRecord{f.old, f.next} {
if err := f.db.UpsertWorkflow(f.ctx, workflow); err != nil {
t.Fatal(err)
}
}
if err := f.db.UpsertOperator(f.ctx, domain.OperatorRecord{ID: "operator-builtin-package", Type: "builtin", Image: "registry.example/package:catalog", ImageID: "registry.example/package@sha256:" + strings.Repeat("c", 64)}); err != nil {
t.Fatal(err)
}
f.runtime = &Runtime{store: f.db}
return f
}
func (f *captureCutoverFixture) seed(name string, status domain.CaptureRecordStatus, runID string) domain.CaptureRecord {
f.t.Helper()
groups, err := discoverCaptureGroups(f.objects, normalizedCaptureAutomation(f.old.CaptureAutomation))
if err != nil {
f.t.Fatal(err)
}
var selected captureGroup
for _, group := range groups {
if group.CaptureID == name+"/part0001" {
selected = group
}
}
if selected.Prefix == "" || len(selected.Missing) != 0 {
f.t.Fatalf("invalid cutover source fixture: %+v", selected)
}
c := domain.CaptureRecord{ID: captureRecordID(f.old.ID, selected.Prefix), OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, ConnectionID: "cutover-storage", Bucket: "synthetic", CaptureID: selected.CaptureID, Prefix: selected.Prefix, SourcePrefix: selected.SourcePrefix, SourceKeys: selected.SourceKeys, Fingerprint: selected.Fingerprint, Status: status, RunID: runID, StabilityChecks: 1, StabilityObservedAt: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano), DiscoveredAt: "2026-01-01T00:00:00Z", UpdatedAt: "2026-01-01T00:00:00Z", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "cutover-storage", Bucket: "synthetic", Prefix: "deliveries/cutover__project/" + selected.CaptureID}}
if err := f.db.UpsertCaptureRecord(f.ctx, c); err != nil {
f.t.Fatal(err)
}
saved, err := f.db.GetCaptureRecord(f.ctx, c.ID)
if err != nil {
f.t.Fatal(err)
}
return *saved
}
func (f *captureCutoverFixture) activate() {
f.t.Helper()
w := httptest.NewRecorder()
api.NewHandlerWithConfig(f.db, nil, config.AppConfig{}).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/workflows/"+f.next.ID+"/activate", nil))
if w.Code != http.StatusOK {
f.t.Fatalf("activation: %d %s", w.Code, w.Body.String())
}
}
func (f *captureCutoverFixture) scan() {
f.t.Helper()
if err := f.db.DeleteSourceStatesByWorkflow(f.ctx, f.next.ID); err != nil {
f.t.Fatal(err)
}
w, err := f.db.GetWorkflow(f.ctx, f.next.ID)
if err != nil {
f.t.Fatal(err)
}
if err := f.runtime.processCaptureAutomation(f.ctx, *w); err != nil {
f.t.Fatal(err)
}
}
func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testing.T) {
f := newCaptureCutoverFixture(t, "active", "done", "launching", "fresh")
active := f.seed("active", domain.CaptureRecordWaiting, "old-parent")
done := f.seed("done", domain.CaptureRecordCompleted, "delivered-parent")
launching := f.seed("launching", domain.CaptureRecordLaunching, "old-claimed")
fresh := f.seed("fresh", domain.CaptureRecordStabilizing, "")
parent := domain.RunRecord{ID: active.RunID, OrganizationID: active.OrganizationID, ProjectID: active.ProjectID, WorkflowID: f.old.ID, TaskID: active.ID, Status: domain.RunStatusWaitingForHuman, CreatedAt: "2026-01-01T00:00:00Z"}
child := parent
child.ID, child.ParentRunID, child.Trigger, child.Status, child.CreatedAt = "old-child", parent.ID, domain.RunTriggerHumanRelease, domain.RunStatusRunning, "2026-01-01T00:00:01Z"
for _, run := range []domain.RunRecord{parent, child, {ID: launching.RunID, OrganizationID: launching.OrganizationID, ProjectID: launching.ProjectID, WorkflowID: f.old.ID, TaskID: launching.ID, Status: domain.RunStatusQueued}} {
if err := f.db.UpsertRun(f.ctx, run); err != nil {
t.Fatal(err)
}
}
publisher := &deliveryPublisher{}
f.runtime.artifactPublisher = publisher
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil {
t.Fatal(err)
}
if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil {
t.Fatalf("old child cannot publish: %v", err)
}
f.activate()
if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil {
t.Fatalf("completed legacy recovery failed: %v", err)
}
for _, before := range []domain.CaptureRecord{active, done, launching} {
after, _ := f.db.GetCaptureRecord(f.ctx, before.ID)
if after == nil || after.WorkflowID != f.old.ID || after.RunID != before.RunID {
t.Fatalf("started Capture moved: before=%+v after=%+v", before, after)
}
}
lookup, err := f.db.GetCaptureRecordByRunID(f.ctx, child.ID)
if err != nil || lookup == nil || lookup.ID != active.ID {
t.Fatalf("old child lost Capture: %+v %v", lookup, err)
}
child.Status = domain.RunStatusCompleted
if err := f.db.UpsertRun(f.ctx, child); err != nil {
t.Fatal(err)
}
f.scan()
oldAfter, _ := f.db.GetCaptureRecord(f.ctx, active.ID)
if oldAfter.WorkflowID != f.old.ID || oldAfter.RunID != child.ID || oldAfter.Status != domain.CaptureRecordCompleted {
t.Fatalf("hidden lineage did not reconcile: %+v", oldAfter)
}
if lookup, err = f.db.GetCaptureRecordByRunID(f.ctx, parent.ID); err != nil || lookup == nil || lookup.ID != active.ID {
t.Fatalf("old parent lost child-owned Capture: %+v %v", lookup, err)
}
f.scan()
newCapture, _ := f.db.GetCaptureRecord(f.ctx, fresh.ID)
if newCapture == nil || newCapture.WorkflowID != f.next.ID || newCapture.RunID == "" {
t.Fatalf("unstarted Capture not admitted: %+v", newCapture)
}
newRun, _ := f.db.GetRun(f.ctx, newCapture.RunID)
if newRun.WorkflowID != f.next.ID || newRun.TaskID != fresh.ID {
t.Fatalf("wrong new Run binding: %+v", newRun)
}
oldMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, child.WorkflowID)
if err != nil || !oldMode {
t.Fatalf("old child did not select v4 policy: %v %v", oldMode, err)
}
newMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, newRun.WorkflowID)
if err != nil || !newMode {
t.Fatalf("new parent did not select v4: %v %v", newMode, err)
}
for _, run := range []domain.RunRecord{parent, child, *newRun} {
workflow, err := f.db.GetWorkflow(f.ctx, run.WorkflowID)
if err != nil {
t.Fatal(err)
}
resolved, err := f.runtime.resolveBuiltinNodeRuntime(f.ctx, workflow.Nodes[0])
want := f.old.Nodes[0]
if run.WorkflowID == f.next.ID {
want = f.next.Nodes[0]
}
if err != nil || resolved.Data.Image != want.Data.Image || resolved.Data.OperatorImageID != want.Data.OperatorImageID {
t.Fatalf("Run %s lost pinned original runtime: got=%+v want=%+v err=%v", run.ID, resolved.Data, want.Data, err)
}
}
completed, _ := f.db.GetCaptureRecord(f.ctx, done.ID)
if !reflect.DeepEqual(*completed, done) {
t.Fatalf("delivered Capture changed: before=%+v after=%+v", done, completed)
}
all, _ := f.db.ListCaptureRecords(f.ctx, f.old.OrganizationID, "", 0)
if len(all) != 4 {
t.Fatalf("Capture re-ingestion or ID collision: %+v", all)
}
}
func TestCaptureCutoverMigratedUnstartedKeepsIdentity(t *testing.T) {
f := newCaptureCutoverFixture(t, "fresh")
before := f.seed("fresh", domain.CaptureRecordStabilizing, "")
f.activate()
f.scan()
f.scan()
after, _ := f.db.GetCaptureRecord(f.ctx, before.ID)
all, _ := f.db.ListCaptureRecords(f.ctx, f.next.OrganizationID, f.next.ID, 0)
if after == nil || after.RunID == "" || len(all) != 1 || all[0].ID != before.ID {
t.Fatalf("migrated identity lost/duplicated: %+v %+v", after, all)
}
}
func TestCaptureCutoverDoesNotDeduplicateOtherScopes(t *testing.T) {
for _, scope := range []string{"project", "organization", "family"} {
t.Run(scope, func(t *testing.T) {
f := newCaptureCutoverFixture(t, "fresh")
foreign := f.old
foreign.ID = "foreign-workflow"
switch scope {
case "project":
foreign.ProjectID = "other-project"
case "organization":
foreign.OrganizationID = "other-org"
case "family":
foreign.DefinitionFamilyID = "other-family"
}
if err := f.db.UpsertWorkflow(f.ctx, foreign); err != nil {
t.Fatal(err)
}
original := f.seed("fresh", domain.CaptureRecordStabilizing, "")
other := original
other.ID, other.WorkflowID, other.OrganizationID, other.ProjectID, other.RunID, other.Status = "foreign-capture", foreign.ID, foreign.OrganizationID, foreign.ProjectID, "foreign-run", domain.CaptureRecordCompleted
if err := f.db.UpsertCaptureRecord(f.ctx, other); err != nil {
t.Fatal(err)
}
f.activate()
f.scan()
got, _ := f.db.GetCaptureRecord(f.ctx, original.ID)
untouched, _ := f.db.GetCaptureRecord(f.ctx, other.ID)
if got.RunID == "" || got.WorkflowID != f.next.ID || untouched.WorkflowID != foreign.ID || untouched.RunID != other.RunID {
t.Fatalf("scope crossed: own=%+v foreign=%+v", got, untouched)
}
})
}
}
func TestCaptureCutoverStopsScanCapturedBeforeActivation(t *testing.T) {
f := newCaptureCutoverFixture(t, "fresh")
before := f.seed("fresh", domain.CaptureRecordStabilizing, "")
f.beforeList = f.activate
if err := f.runtime.processCaptureAutomation(f.ctx, f.old); err != nil {
t.Fatal(err)
}
after, _ := f.db.GetCaptureRecord(f.ctx, before.ID)
if after.WorkflowID != f.next.ID || after.RunID != "" {
t.Fatalf("stale old scan wrote/admitted after activation: %+v", after)
}
}
func TestCaptureCutoverReadsAllHiddenAncestorsAndReconcilesMissingSources(t *testing.T) {
f := newCaptureCutoverFixture(t, "ancient", "missing", "claim")
ancestor := f.old
ancestor.ID, ancestor.Hidden = "hidden-ancestor", true
f.old.SupersedesWorkflowID, f.old.DefinitionVersion = ancestor.ID, 2
f.next.DefinitionVersion = 3
for _, workflow := range []domain.WorkflowRecord{ancestor, f.old, f.next} {
if err := f.db.UpsertWorkflow(f.ctx, workflow); err != nil {
t.Fatal(err)
}
}
ancient := f.seed("ancient", domain.CaptureRecordCompleted, "delivered-ancestor")
missing := f.seed("missing", domain.CaptureRecordWaiting, "ancestor-run")
claim := f.seed("claim", domain.CaptureRecordLaunching, "")
for _, record := range []*domain.CaptureRecord{&ancient, &missing, &claim} {
record.WorkflowID = ancestor.ID
if err := f.db.UpsertCaptureRecord(f.ctx, *record); err != nil {
t.Fatal(err)
}
}
for _, run := range []domain.RunRecord{
{ID: missing.RunID, OrganizationID: ancestor.OrganizationID, ProjectID: ancestor.ProjectID, WorkflowID: ancestor.ID, TaskID: missing.ID, Status: domain.RunStatusCompleted},
// The Worker died after CreateRun but before recording its ID on Capture.
{ID: "claimed-old-run", OrganizationID: ancestor.OrganizationID, ProjectID: ancestor.ProjectID, WorkflowID: ancestor.ID, TaskID: claim.ID, Status: domain.RunStatusQueued},
} {
if err := f.db.UpsertRun(f.ctx, run); err != nil {
t.Fatal(err)
}
}
filtered := []captureObject{}
for _, object := range f.objects {
if !strings.Contains(object.Key, "/missing/") {
filtered = append(filtered, object)
}
}
f.objects = filtered
f.activate()
f.scan()
f.scan()
completed, _ := f.db.GetCaptureRecord(f.ctx, ancient.ID)
if !reflect.DeepEqual(*completed, ancient) {
t.Fatalf("historical delivery changed: %+v", completed)
}
reconciled, _ := f.db.GetCaptureRecord(f.ctx, missing.ID)
if reconciled.WorkflowID != ancestor.ID || reconciled.Status != domain.CaptureRecordCompleted {
t.Fatalf("hidden ancestor without source was not reconciled: %+v", reconciled)
}
recovered, _ := f.db.GetCaptureRecord(f.ctx, claim.ID)
if recovered.WorkflowID != ancestor.ID || recovered.RunID != "claimed-old-run" || recovered.Status != domain.CaptureRecordQueued {
t.Fatalf("interrupted old launch lost its owner: %+v", recovered)
}
all, _ := f.db.ListCaptureRecords(f.ctx, f.old.OrganizationID, "", 0)
if len(all) != 3 {
t.Fatalf("hidden ancestor re-ingested: %+v", all)
}
}
func TestCaptureCutoverTransfersOnlyNeverStartedStates(t *testing.T) {
states := []domain.CaptureRecordStatus{domain.CaptureRecordIncomplete, domain.CaptureRecordStabilizing, domain.CaptureRecordReady, domain.CaptureRecordLaunching, domain.CaptureRecordRequeueing, domain.CaptureRecordQueued, domain.CaptureRecordRunning, domain.CaptureRecordWaiting, domain.CaptureRecordCompleted, domain.CaptureRecordFailed, domain.CaptureRecordCancelled, domain.CaptureRecordDiscarded}
names := []string{}
for _, state := range states {
names = append(names, string(state))
}
f := newCaptureCutoverFixture(t, names...)
before := []domain.CaptureRecord{}
for _, state := range states {
runID := "old-" + string(state)
if state == domain.CaptureRecordIncomplete || state == domain.CaptureRecordStabilizing || state == domain.CaptureRecordReady || state == domain.CaptureRecordRequeueing || state == domain.CaptureRecordDiscarded {
runID = ""
}
before = append(before, f.seed(string(state), state, runID))
}
f.activate()
for _, previous := range before {
after, _ := f.db.GetCaptureRecord(f.ctx, previous.ID)
move := previous.Status == domain.CaptureRecordIncomplete || previous.Status == domain.CaptureRecordStabilizing || previous.Status == domain.CaptureRecordReady
if move {
if after.WorkflowID != f.next.ID || after.RunID != "" || after.ID != previous.ID {
t.Fatalf("unstarted transfer: %+v", after)
}
} else if !reflect.DeepEqual(*after, previous) {
t.Fatalf("historical/claimed record changed: before=%+v after=%+v", previous, after)
}
}
}
// Store faults exercise partial activation and unknown Run creation outcomes.
type captureCutoverFaultStore struct {
store.Store
upsertWorkflow func(context.Context, domain.WorkflowRecord) error
getRun func(context.Context, string, string) (*domain.RunRecord, error)
createRun func(context.Context, string, domain.RunTrigger, map[string]domain.ArtifactPayload, ...string) (*domain.RunRecord, error)
}
func (s *captureCutoverFaultStore) UpsertWorkflow(ctx context.Context, w domain.WorkflowRecord) error {
if s.upsertWorkflow != nil {
return s.upsertWorkflow(ctx, w)
}
return s.Store.UpsertWorkflow(ctx, w)
}
func (s *captureCutoverFaultStore) GetRunByWorkflowTaskID(ctx context.Context, workflowID, taskID string) (*domain.RunRecord, error) {
if s.getRun != nil {
return s.getRun(ctx, workflowID, taskID)
}
return s.Store.GetRunByWorkflowTaskID(ctx, workflowID, taskID)
}
func (s *captureCutoverFaultStore) CreateRunWithInputs(ctx context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) {
if s.createRun != nil {
return s.createRun(ctx, workflowID, trigger, inputs, taskIDs...)
}
return s.Store.CreateRunWithInputs(ctx, workflowID, trigger, inputs, taskIDs...)
}
func (f *captureCutoverFixture) activateResponse() *httptest.ResponseRecorder {
w := httptest.NewRecorder()
api.NewHandlerWithConfig(f.runtime.store, nil, config.AppConfig{}).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/api/workflows/"+f.next.ID+"/activate", nil))
return w
}
func TestCaptureCutoverRetryTransfersHiddenUnstartedAndRollsBackOnlyOwnWrites(t *testing.T) {
f := newCaptureCutoverFixture(t, "stranded", "already-moved")
stranded := f.seed("stranded", domain.CaptureRecordReady, "")
already := f.seed("already-moved", domain.CaptureRecordReady, "")
already.WorkflowID = f.next.ID
if err := f.db.UpsertCaptureRecord(f.ctx, already); err != nil {
t.Fatal(err)
}
// Persisted state after an earlier process died after hiding the old owner.
f.old.Hidden = true
if err := f.db.UpsertWorkflow(f.ctx, f.old); err != nil {
t.Fatal(err)
}
fault := &captureCutoverFaultStore{Store: f.db}
fault.upsertWorkflow = func(ctx context.Context, w domain.WorkflowRecord) error {
if w.ID == f.next.ID && !w.Hidden {
return fmt.Errorf("injected activation write failure")
}
return f.db.UpsertWorkflow(ctx, w)
}
f.runtime.store = fault
if got := f.activateResponse(); got.Code != http.StatusInternalServerError {
t.Fatalf("expected injected failure: %d %s", got.Code, got.Body.String())
}
old, _ := f.db.GetWorkflow(f.ctx, f.old.ID)
got, _ := f.db.GetCaptureRecord(f.ctx, stranded.ID)
untouched, _ := f.db.GetCaptureRecord(f.ctx, already.ID)
if !old.Hidden || !reflect.DeepEqual(*got, stranded) || !reflect.DeepEqual(*untouched, already) {
t.Fatalf("rollback changed prior request state: old=%+v stranded=%+v moved=%+v", old, got, untouched)
}
f.runtime.store = f.db
f.activate()
got, _ = f.db.GetCaptureRecord(f.ctx, stranded.ID)
if got.WorkflowID != f.next.ID || got.ID != stranded.ID {
t.Fatalf("hidden unstarted capture stranded: %+v", got)
}
f.scan()
got, _ = f.db.GetCaptureRecord(f.ctx, stranded.ID)
if got.RunID == "" {
t.Fatalf("transferred hidden capture not admitted: %+v", got)
}
}
func TestCaptureCutoverUnresolvedClaimBlocksBeforeAnyWrites(t *testing.T) {
for _, hidden := range []bool{false, true} {
for _, lookupFails := range []bool{false, true} {
t.Run(fmt.Sprintf("hidden=%v/lookupFails=%v", hidden, lookupFails), func(t *testing.T) {
f := newCaptureCutoverFixture(t, "claim", "fresh")
claim := f.seed("claim", domain.CaptureRecordLaunching, "")
fresh := f.seed("fresh", domain.CaptureRecordReady, "")
f.old.Hidden = hidden
if err := f.db.UpsertWorkflow(f.ctx, f.old); err != nil {
t.Fatal(err)
}
if lookupFails {
f.runtime.store = &captureCutoverFaultStore{Store: f.db, getRun: func(context.Context, string, string) (*domain.RunRecord, error) {
return nil, fmt.Errorf("injected Run lookup failure")
}}
}
got := f.activateResponse()
want := http.StatusConflict
if lookupFails {
want = http.StatusInternalServerError
}
if got.Code != want || (!lookupFails && !strings.Contains(got.Body.String(), "capture_launch_unresolved")) {
t.Fatalf("unsafe activation: %d %s", got.Code, got.Body.String())
}
for _, before := range []domain.WorkflowRecord{f.old, f.next} {
after, _ := f.db.GetWorkflow(f.ctx, before.ID)
if !reflect.DeepEqual(*after, before) {
t.Fatalf("preflight wrote workflow: %+v", after)
}
}
for _, before := range []domain.CaptureRecord{claim, fresh} {
after, _ := f.db.GetCaptureRecord(f.ctx, before.ID)
if !reflect.DeepEqual(*after, before) {
t.Fatalf("preflight wrote Capture: %+v", after)
}
}
})
}
}
}
func TestCaptureCutoverScannerSurfacesUnresolvedClaimWithoutRewriting(t *testing.T) {
for _, hidden := range []bool{false, true} {
t.Run(fmt.Sprintf("hidden=%v", hidden), func(t *testing.T) {
f := newCaptureCutoverFixture(t, "claim")
claim := f.seed("claim", domain.CaptureRecordLaunching, "")
owner := f.old
if hidden {
f.old.Hidden, f.next.Hidden = true, false
for _, w := range []domain.WorkflowRecord{f.old, f.next} {
if err := f.db.UpsertWorkflow(f.ctx, w); err != nil {
t.Fatal(err)
}
}
owner = f.next
}
err := f.runtime.processCaptureAutomation(f.ctx, owner)
if err == nil || !strings.Contains(err.Error(), "capture_launch_unresolved") || !strings.Contains(err.Error(), claim.ID) || !strings.Contains(err.Error(), f.old.ID) {
t.Fatalf("unresolved claim silently skipped: %v", err)
}
after, _ := f.db.GetCaptureRecord(f.ctx, claim.ID)
runs, _ := f.db.ListRuns(f.ctx, "")
if !reflect.DeepEqual(*after, claim) || len(runs) != 0 {
t.Fatalf("unfenced recovery: Capture=%+v runs=%+v", after, runs)
}
})
}
}
func TestCaptureCutoverUnknownCreateOutcomeNeverReopensAdmission(t *testing.T) {
for _, persisted := range []bool{false, true} {
t.Run(fmt.Sprintf("persisted=%v", persisted), func(t *testing.T) {
f := newCaptureCutoverFixture(t, "claim")
claim := f.seed("claim", domain.CaptureRecordReady, "")
calls := 0
f.runtime.store = &captureCutoverFaultStore{Store: f.db, createRun: func(ctx context.Context, id string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) {
calls++
if persisted {
if _, err := f.db.CreateRunWithInputs(ctx, id, trigger, inputs, taskIDs...); err != nil {
t.Fatal(err)
}
}
return nil, fmt.Errorf("injected lost CreateRun response")
}}
err := f.runtime.launchCaptureRun(f.ctx, f.old, normalizedCaptureAutomation(f.old.CaptureAutomation), &claim)
if !persisted && (err == nil || !strings.Contains(err.Error(), "capture_launch_unresolved")) {
t.Fatalf("missing actionable diagnostic: %v", err)
}
after, _ := f.db.GetCaptureRecord(f.ctx, claim.ID)
if persisted {
if after.RunID == "" {
t.Fatalf("lost response not reconciled: %+v (%v)", after, err)
}
} else if after.Status != domain.CaptureRecordLaunching || after.RunID != "" {
t.Fatalf("unknown outcome reopened admission: %+v", after)
}
_ = f.runtime.processCaptureAutomation(f.ctx, f.old)
if calls != 1 {
t.Fatalf("duplicated unknown CreateRun: %d", calls)
}
})
}
}
func TestCaptureCutoverResolvesDuplicateUnstartedFamilyPrefixRecords(t *testing.T) {
f := newCaptureCutoverFixture(t, "dup-started", "dup-unstarted", "fresh")
started := f.seed("dup-started", domain.CaptureRecordCompleted, "old-delivered")
// An unstarted duplicate for the same prefix was migrated to the active workflow.
startedDup := started
startedDup.ID = captureRecordID(f.next.ID, started.Prefix)
startedDup.WorkflowID = f.next.ID
startedDup.Status = domain.CaptureRecordStabilizing
startedDup.RunID = ""
if err := f.db.UpsertCaptureRecord(f.ctx, startedDup); err != nil {
t.Fatal(err)
}
unstartedOld := f.seed("dup-unstarted", domain.CaptureRecordStabilizing, "")
unstartedNew := unstartedOld
unstartedNew.ID = captureRecordID(f.next.ID, unstartedOld.Prefix)
unstartedNew.WorkflowID = f.next.ID
if err := f.db.UpsertCaptureRecord(f.ctx, unstartedNew); err != nil {
t.Fatal(err)
}
fresh := f.seed("fresh", domain.CaptureRecordStabilizing, "")
f.activate()
f.scan()
f.scan()
// The started record must stay with its original owner and prevent the unstarted duplicate from launching a second Run.
gotStarted, _ := f.db.GetCaptureRecord(f.ctx, started.ID)
gotStartedDup, _ := f.db.GetCaptureRecord(f.ctx, startedDup.ID)
if gotStarted == nil || gotStarted.WorkflowID != f.old.ID || gotStarted.RunID != "old-delivered" {
t.Fatalf("started record altered: %+v", gotStarted)
}
if gotStartedDup == nil || gotStartedDup.RunID != "" {
t.Fatalf("unstarted duplicate of started prefix launched a Run: %+v", gotStartedDup)
}
// The unstarted duplicate on the active workflow should advance and launch without aborting the scan.
gotUnstartedNew, _ := f.db.GetCaptureRecord(f.ctx, unstartedNew.ID)
if gotUnstartedNew == nil || gotUnstartedNew.WorkflowID != f.next.ID || gotUnstartedNew.RunID == "" {
t.Fatalf("active unstarted record did not launch: %+v", gotUnstartedNew)
}
gotFresh, _ := f.db.GetCaptureRecord(f.ctx, fresh.ID)
if gotFresh == nil || gotFresh.WorkflowID != f.next.ID || gotFresh.RunID == "" {
t.Fatalf("fresh record blocked by family prefix duplicate: %+v", gotFresh)
}
}