d-robotics dc81aaf5fe 🐛 Preserve deployed worker retry protections before scheduler rollout
(cherry picked from commit 6f8b6a7f01bfdbb55f501c60e121ede64d52c5a4)
2026-09-29 08:54:35 -07:00

150 lines
5.8 KiB
Go

package worker
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func TestRetryPreservesHumanReleaseBaseline(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore()
wf, err := s.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "retry", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "gate", Target: "package"}}})
if err != nil {
t.Fatal(err)
}
run := domain.RunRecord{ID: "retry-release", WorkflowID: wf.ID, Trigger: domain.RunTriggerHumanRelease, RestartFromNodeID: "package", Status: domain.RunStatusFailed, Attempt: 1, NodeExecutions: []domain.NodeExecutionRecord{
{NodeID: "gate", Status: domain.NodeExecutionStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: "accepted", Path: "/workspace/accepted.json"}}},
{NodeID: "package", Status: domain.NodeExecutionStatusFailed, Error: "old image"},
}}
if err := s.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
h := NewHTTPHandler(s, &Runtime{store: s})
r := httptest.NewRecorder()
h.ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/retry-release/retry", nil))
if r.Code != http.StatusAccepted {
t.Fatalf("retry: %d %s", r.Code, r.Body.String())
}
got, _ := s.GetRun(ctx, run.ID)
if len(got.NodeExecutions) != 1 || got.NodeExecutions[0].NodeID != "gate" || len(got.NodeExecutions[0].Artifacts) != 1 || got.NodeExecutions[0].Artifacts[0].Path != "/workspace/accepted.json" {
t.Fatalf("accepted baseline lost: %#v", got.NodeExecutions)
}
if len(got.RetryHistory) != 1 || got.RetryHistory[0].NodeExecutions[1].Error != "old image" {
t.Fatal("failed attempt evidence lost")
}
if strings.Contains(r.Body.String(), "retryHistory") {
t.Fatal("private execution history leaked")
}
executions := map[string]domain.NodeExecutionRecord{"gate": got.NodeExecutions[0]}
if !dependenciesComplete(*wf, executions, "package") {
t.Fatal("package dependency remains blocked")
}
inputs, _, _ := gatherNodeInputs(*wf, got, executions, "package")
if len(inputs) != 1 || inputs[0].Path != "/workspace/accepted.json" {
t.Fatalf("package inputs=%#v", inputs)
}
if got.Attempt != 2 {
t.Fatalf("attempt = %d", got.Attempt)
}
r = httptest.NewRecorder()
h.ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/retry-release/retry", nil))
if r.Code != http.StatusConflict {
t.Fatalf("duplicate retry: %d", r.Code)
}
}
func TestRetryRejectsActiveAndCompletedRuns(t *testing.T) {
for _, status := range []domain.RunStatus{domain.RunStatusQueued, domain.RunStatusRunning, domain.RunStatusCompleted, domain.RunStatusWaitingForAudit} {
t.Run(string(status), func(t *testing.T) {
s := store.NewMemoryStore()
if err := s.UpsertRun(context.Background(), domain.RunRecord{ID: "run", Status: status}); err != nil {
t.Fatal(err)
}
r := httptest.NewRecorder()
NewHTTPHandler(s, &Runtime{store: s}).ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/run/retry", nil))
if r.Code != http.StatusConflict {
t.Fatalf("status=%d", r.Code)
}
})
}
}
func TestRetryRejectsMissingBaselineAndActiveLease(t *testing.T) {
for _, leased := range []bool{false, true} {
s := store.NewMemoryStore()
ctx := context.Background()
wf, err := s.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "guard", Nodes: []domain.WorkflowNode{{ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "gate", Target: "package"}}})
if err != nil {
t.Fatal(err)
}
run := domain.RunRecord{ID: "guard", WorkflowID: wf.ID, Status: domain.RunStatusFailed, Trigger: domain.RunTriggerHumanRelease, RestartFromNodeID: "package", Error: "original"}
if err := s.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
if leased {
if _, ok, err := s.AcquireRunLease(ctx, run.ID, "active-worker", time.Minute); err != nil || !ok {
t.Fatal("lease")
}
}
r := httptest.NewRecorder()
NewHTTPHandler(s, &Runtime{store: s}).ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/guard/retry", nil))
if r.Code != 409 {
t.Fatalf("leased=%v: %d %s", leased, r.Code, r.Body.String())
}
got, _ := s.GetRun(ctx, run.ID)
if got.Status != domain.RunStatusFailed || got.Error != "original" {
t.Fatal("rejected retry changed failed run")
}
}
}
func TestRetryConcurrentRequestsQueueOnlyOnce(t *testing.T) {
s := store.NewMemoryStore()
ctx := context.Background()
wf, err := s.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "retry", Nodes: []domain.WorkflowNode{{ID: "task"}}})
if err != nil {
t.Fatal(err)
}
run := domain.RunRecord{ID: "concurrent", WorkflowID: wf.ID, Status: domain.RunStatusCancelled, StopMode: domain.RunStopModeForce, StopRequestedAt: nowISO(), StartedAt: nowISO(), NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "task", Attempt: 5, Status: domain.NodeExecutionStatusCancelled}}}
if err := s.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
h := NewHTTPHandler(s, &Runtime{store: s})
codes := make(chan int, 10)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
r := httptest.NewRecorder()
h.ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/concurrent/retry", nil))
codes <- r.Code
}()
}
wg.Wait()
close(codes)
accepted := 0
for code := range codes {
if code == 202 {
accepted++
} else if code != 409 {
t.Fatalf("unexpected %d", code)
}
}
if accepted != 1 {
t.Fatalf("accepted %d retries", accepted)
}
got, _ := s.GetRun(ctx, run.ID)
if got.Attempt != 6 || len(got.RetryHistory) != 1 || got.StopMode != "" || got.StopRequestedAt != "" || got.StartedAt != "" {
t.Fatalf("bad retry lifecycle: attempt=%d history=%d stop=%s started=%s", got.Attempt, len(got.RetryHistory), got.StopMode, got.StartedAt)
}
}