forked from longtao.wu/droboticflow
150 lines
5.8 KiB
Go
150 lines
5.8 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func TestRetryPreservesHumanReleaseBaseline(t *testing.T) {
|
|
ctx := context.Background()
|
|
s := store.NewMemoryStore()
|
|
wf, err := s.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "retry", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "gate", Target: "package"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run := domain.RunRecord{ID: "retry-release", WorkflowID: wf.ID, Trigger: domain.RunTriggerHumanRelease, RestartFromNodeID: "package", Status: domain.RunStatusFailed, Attempt: 1, NodeExecutions: []domain.NodeExecutionRecord{
|
|
{NodeID: "gate", Status: domain.NodeExecutionStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: "accepted", Path: "/workspace/accepted.json"}}},
|
|
{NodeID: "package", Status: domain.NodeExecutionStatusFailed, Error: "old image"},
|
|
}}
|
|
if err := s.UpsertRun(ctx, run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := NewHTTPHandler(s, &Runtime{store: s})
|
|
r := httptest.NewRecorder()
|
|
h.ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/retry-release/retry", nil))
|
|
if r.Code != http.StatusAccepted {
|
|
t.Fatalf("retry: %d %s", r.Code, r.Body.String())
|
|
}
|
|
got, _ := s.GetRun(ctx, run.ID)
|
|
if len(got.NodeExecutions) != 1 || got.NodeExecutions[0].NodeID != "gate" || len(got.NodeExecutions[0].Artifacts) != 1 || got.NodeExecutions[0].Artifacts[0].Path != "/workspace/accepted.json" {
|
|
t.Fatalf("accepted baseline lost: %#v", got.NodeExecutions)
|
|
}
|
|
if len(got.RetryHistory) != 1 || got.RetryHistory[0].NodeExecutions[1].Error != "old image" {
|
|
t.Fatal("failed attempt evidence lost")
|
|
}
|
|
if strings.Contains(r.Body.String(), "retryHistory") {
|
|
t.Fatal("private execution history leaked")
|
|
}
|
|
executions := map[string]domain.NodeExecutionRecord{"gate": got.NodeExecutions[0]}
|
|
if !dependenciesComplete(*wf, executions, "package") {
|
|
t.Fatal("package dependency remains blocked")
|
|
}
|
|
inputs, _, _ := gatherNodeInputs(*wf, got, executions, "package")
|
|
if len(inputs) != 1 || inputs[0].Path != "/workspace/accepted.json" {
|
|
t.Fatalf("package inputs=%#v", inputs)
|
|
}
|
|
if got.Attempt != 2 {
|
|
t.Fatalf("attempt = %d", got.Attempt)
|
|
}
|
|
r = httptest.NewRecorder()
|
|
h.ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/retry-release/retry", nil))
|
|
if r.Code != http.StatusConflict {
|
|
t.Fatalf("duplicate retry: %d", r.Code)
|
|
}
|
|
}
|
|
|
|
func TestRetryRejectsActiveAndCompletedRuns(t *testing.T) {
|
|
for _, status := range []domain.RunStatus{domain.RunStatusQueued, domain.RunStatusRunning, domain.RunStatusCompleted, domain.RunStatusWaitingForAudit} {
|
|
t.Run(string(status), func(t *testing.T) {
|
|
s := store.NewMemoryStore()
|
|
if err := s.UpsertRun(context.Background(), domain.RunRecord{ID: "run", Status: status}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := httptest.NewRecorder()
|
|
NewHTTPHandler(s, &Runtime{store: s}).ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/run/retry", nil))
|
|
if r.Code != http.StatusConflict {
|
|
t.Fatalf("status=%d", r.Code)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetryRejectsMissingBaselineAndActiveLease(t *testing.T) {
|
|
for _, leased := range []bool{false, true} {
|
|
s := store.NewMemoryStore()
|
|
ctx := context.Background()
|
|
wf, err := s.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "guard", Nodes: []domain.WorkflowNode{{ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "gate", Target: "package"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run := domain.RunRecord{ID: "guard", WorkflowID: wf.ID, Status: domain.RunStatusFailed, Trigger: domain.RunTriggerHumanRelease, RestartFromNodeID: "package", Error: "original"}
|
|
if err := s.UpsertRun(ctx, run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if leased {
|
|
if _, ok, err := s.AcquireRunLease(ctx, run.ID, "active-worker", time.Minute); err != nil || !ok {
|
|
t.Fatal("lease")
|
|
}
|
|
}
|
|
r := httptest.NewRecorder()
|
|
NewHTTPHandler(s, &Runtime{store: s}).ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/guard/retry", nil))
|
|
if r.Code != 409 {
|
|
t.Fatalf("leased=%v: %d %s", leased, r.Code, r.Body.String())
|
|
}
|
|
got, _ := s.GetRun(ctx, run.ID)
|
|
if got.Status != domain.RunStatusFailed || got.Error != "original" {
|
|
t.Fatal("rejected retry changed failed run")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryConcurrentRequestsQueueOnlyOnce(t *testing.T) {
|
|
s := store.NewMemoryStore()
|
|
ctx := context.Background()
|
|
wf, err := s.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "retry", Nodes: []domain.WorkflowNode{{ID: "task"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run := domain.RunRecord{ID: "concurrent", WorkflowID: wf.ID, Status: domain.RunStatusCancelled, StopMode: domain.RunStopModeForce, StopRequestedAt: nowISO(), StartedAt: nowISO(), NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "task", Attempt: 5, Status: domain.NodeExecutionStatusCancelled}}}
|
|
if err := s.UpsertRun(ctx, run); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := NewHTTPHandler(s, &Runtime{store: s})
|
|
codes := make(chan int, 10)
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 10; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
r := httptest.NewRecorder()
|
|
h.ServeHTTP(r, httptest.NewRequest(http.MethodPost, "/worker-api/runs/concurrent/retry", nil))
|
|
codes <- r.Code
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
close(codes)
|
|
accepted := 0
|
|
for code := range codes {
|
|
if code == 202 {
|
|
accepted++
|
|
} else if code != 409 {
|
|
t.Fatalf("unexpected %d", code)
|
|
}
|
|
}
|
|
if accepted != 1 {
|
|
t.Fatalf("accepted %d retries", accepted)
|
|
}
|
|
got, _ := s.GetRun(ctx, run.ID)
|
|
if got.Attempt != 6 || len(got.RetryHistory) != 1 || got.StopMode != "" || got.StopRequestedAt != "" || got.StartedAt != "" {
|
|
t.Fatalf("bad retry lifecycle: attempt=%d history=%d stop=%s started=%s", got.Attempt, len(got.RetryHistory), got.StopMode, got.StartedAt)
|
|
}
|
|
}
|