droboticflow/internal/worker/runtime_test.go

1339 lines
44 KiB
Go

package worker
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func localRunnerCommand(t *testing.T) string {
t.Helper()
path, err := filepath.Abs("../../backend/operator-runner/main.py")
if err != nil {
t.Fatalf("resolve operator runner: %v", err)
}
return "python3 " + path
}
func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord {
t.Helper()
for _, item := range run.NodeExecutions {
if item.NodeID == nodeID {
return item
}
}
t.Fatalf("missing node execution for %s", nodeID)
return domain.NodeExecutionRecord{}
}
func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name string) domain.ExecutionArtifact {
t.Helper()
for _, item := range execution.Artifacts {
if item.Name == name {
return item
}
}
t.Fatalf("missing artifact %s on node %s", name, execution.NodeID)
return domain.ExecutionArtifact{}
}
func writeFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
func createRosbagDB3(t *testing.T, path string) {
t.Helper()
command := exec.Command(
"python3",
"-c",
strings.Join([]string{
"import sqlite3, sys",
"conn = sqlite3.connect(sys.argv[1])",
"cur = conn.cursor()",
"cur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')",
"cur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')",
"cur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")",
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")",
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")",
"conn.commit()",
"conn.close()",
}, "; "),
path,
)
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("create rosbag db3 %s: %v: %s", path, err, string(output))
}
}
func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
if run != nil && run.Status == expected {
return run
}
time.Sleep(100 * time.Millisecond)
}
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
t.Fatalf("timed out waiting for run %s to become %s, got %#v", runID, expected, run)
return nil
}
func waitForRunningNodeCount(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected int, timeout time.Duration) *domain.RunRecord {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
if run == nil {
time.Sleep(100 * time.Millisecond)
continue
}
running := 0
for _, execution := range run.NodeExecutions {
if execution.Status == domain.NodeExecutionStatusRunning {
running++
}
}
if running == expected {
return run
}
time.Sleep(100 * time.Millisecond)
}
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
t.Fatalf("timed out waiting for run %s to have %d running nodes, got %#v", runID, expected, run)
return nil
}
func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Worker 项目", Description: "worker"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "批处理工作流",
Description: "complete path",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "OSS 数据源",
Category: "IO",
Description: "source",
Inputs: []domain.PortDefinition{},
Outputs: []domain.PortDefinition{
{ID: "out-1", Name: "file", Type: domain.PortTypeFile},
},
Hyperparams: []domain.HyperParam{
{Key: "SOURCE_TEXT", Value: "worker-runtime-test"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "sink",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "sink",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{},
},
Position: domain.Position{X: 100, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "e1", Source: "source", SourceHandle: "out-1", Target: "sink", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
for _, item := range completed.NodeExecutions {
if item.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected completed node execution, got %s", item.Status)
}
}
if len(completed.Artifacts) == 0 {
t.Fatalf("expected artifacts")
}
if completed.Artifacts[0].Path == "" {
t.Fatalf("expected real artifact path")
}
if len(completed.ResourceSeries) == 0 {
t.Fatalf("expected resource samples from operator runner")
}
}
func TestRuntimePausesForAuditAndResumes(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Audit 项目", Description: "audit"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "审核工作流",
Description: "audit path",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "review",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "人工审核 (Human Review)",
Category: "HUMAN",
Description: "review",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{
{ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "sink",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{},
},
Position: domain.Position{X: 100, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "e1", Source: "review", SourceHandle: "approved", Target: "upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
paused, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get paused run: %v", err)
}
if paused == nil || paused.Status != domain.RunStatusWaitingForAudit {
t.Fatalf("expected waiting_for_audit run, got %#v", paused)
}
tasks, err := dataStore.ListAuditTasks(ctx, "all")
if err != nil {
t.Fatalf("list audit tasks: %v", err)
}
var task *domain.AuditTaskRecord
for _, item := range tasks {
if item.RunID == run.ID && item.Status == domain.AuditStatusPending {
copied := item
task = &copied
break
}
}
if task == nil {
t.Fatalf("expected pending audit task")
}
task.Status = domain.AuditStatusApproved
task.UpdatedAt = nowISO()
if err := dataStore.UpsertAuditTask(ctx, *task); err != nil {
t.Fatalf("approve audit task: %v", err)
}
paused.Status = domain.RunStatusQueued
paused.UpdatedAt = nowISO()
if err := dataStore.UpsertRun(ctx, *paused); err != nil {
t.Fatalf("queue paused run: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("resume process once: %v", err)
}
resumed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get resumed run: %v", err)
}
if resumed == nil || resumed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed resumed run, got %#v", resumed)
}
}
func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Route 项目", Description: "route"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "条件分流工作流",
Description: "route by handle",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "source",
Outputs: []domain.PortDefinition{
{ID: "out-manifest", Name: "manifest", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "COMMAND", Value: `printf '{"items":[{"kind":"keep","value":1},{"kind":"drop","value":2}]}' > routed.json`},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "route",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "条件分流",
Category: "LOGIC",
Description: "route",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "input", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{
{ID: "matched", Name: "matched", Type: domain.PortTypeJSON},
{ID: "unmatched", Name: "unmatched", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "FIELD_PATH", Value: "kind"},
{Key: "OPERATOR", Value: "eq"},
{Key: "EXPECTED_JSON", Value: `"keep"`},
},
},
Position: domain.Position{X: 200, Y: 0},
},
{
ID: "keep-upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "keep sink",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 420, Y: -80},
},
{
ID: "drop-upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "drop sink",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 420, Y: 80},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-route", Source: "source", SourceHandle: "out-manifest", Target: "route", TargetHandle: "in-1"},
{ID: "route-keep", Source: "route", SourceHandle: "matched", Target: "keep-upload", TargetHandle: "in-1"},
{ID: "route-drop", Source: "route", SourceHandle: "unmatched", Target: "drop-upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
routeExecution := mustNodeExecution(t, completed, "route")
matched := mustArtifact(t, routeExecution, "matched.json")
unmatched := mustArtifact(t, routeExecution, "unmatched.json")
if matched.PortID != "matched" || unmatched.PortID != "unmatched" {
t.Fatalf("expected output port IDs on split artifacts, got matched=%q unmatched=%q", matched.PortID, unmatched.PortID)
}
keepUpload := mustNodeExecution(t, completed, "keep-upload")
dropUpload := mustNodeExecution(t, completed, "drop-upload")
keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json")
dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json")
keepItems, _ := keepSummary.Data["items"].([]any)
dropItems, _ := dropSummary.Data["items"].([]any)
if len(keepItems) != 1 || len(dropItems) != 1 {
t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems))
}
keepPayload, _ := keepItems[0].(map[string]any)
dropPayload, _ := dropItems[0].(map[string]any)
keepRows, _ := keepPayload["items"].([]any)
dropRows, _ := dropPayload["items"].([]any)
if len(keepRows) != 1 || len(dropRows) != 1 {
t.Fatalf("expected one row in each routed payload, got keep=%d drop=%d", len(keepRows), len(dropRows))
}
if keepRow, _ := keepRows[0].(map[string]any); keepRow != nil {
if keepRow["kind"] != "keep" {
t.Fatalf("unexpected keep row: %#v", keepRow)
}
}
if dropRow, _ := dropRows[0].(map[string]any); dropRow != nil {
if dropRow["kind"] != "drop" {
t.Fatalf("unexpected drop row: %#v", dropRow)
}
}
}
func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) {
t.Run("command previews text and csv outputs", func(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Preview 项目", Description: "preview"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "文件预览工作流",
Description: "preview files",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{
{Key: "COMMAND", Value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
commandExecution := mustNodeExecution(t, completed, "command")
report := mustArtifact(t, commandExecution, "report.csv")
notes := mustArtifact(t, commandExecution, "notes.txt")
reportPreview, _ := report.Data["preview"].(map[string]any)
notesPreview, _ := notes.Data["preview"].(map[string]any)
if reportPreview["kind"] != "table" {
t.Fatalf("expected table preview for csv artifact, got %#v", reportPreview)
}
if notesPreview["kind"] != "text" {
t.Fatalf("expected text preview for text artifact, got %#v", notesPreview)
}
})
t.Run("lerobot dataset summary", func(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
datasetRoot := t.TempDir()
writeFile(t, filepath.Join(datasetRoot, "meta", "info.json"), `{"robot_type":"aloha","fps":30}`)
writeFile(t, filepath.Join(datasetRoot, "meta", "stats.json"), `{"episodes":2}`)
writeFile(t, filepath.Join(datasetRoot, "meta", "tasks.jsonl"), "{\"task\":\"pick\"}\n{\"task\":\"place\"}\n")
writeFile(t, filepath.Join(datasetRoot, "data", "episode_000.parquet"), "PAR1")
writeFile(t, filepath.Join(datasetRoot, "videos", "episode_000.mp4"), "video")
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "LeRobot 项目", Description: "lerobot"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "LeRobot 概览",
Description: "lerobot summary",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: datasetRoot}},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "summary",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "LeRobot 数据集概览",
Category: "ROBOTICS",
Description: "lerobot",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 240, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json")
preview, _ := summary.Data["preview"].(map[string]any)
if preview["format"] != "lerobot" {
t.Fatalf("expected lerobot preview, got %#v", preview)
}
if summary.Data["datasetType"] != "lerobot" {
t.Fatalf("expected datasetType lerobot, got %#v", summary.Data)
}
})
t.Run("rosbag sqlite summary", func(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
datasetRoot := t.TempDir()
dbPath := filepath.Join(datasetRoot, "bag_0.db3")
createRosbagDB3(t, dbPath)
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "ROS 项目", Description: "rosbag"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "ROSBag 概览",
Description: "rosbag summary",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "summary",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "ROSBag 数据概览",
Category: "ROBOTICS",
Description: "rosbag",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}},
},
Position: domain.Position{X: 240, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json")
preview, _ := summary.Data["preview"].(map[string]any)
if preview["format"] != "rosbag2" {
t.Fatalf("expected rosbag2 preview, got %#v", preview)
}
topics, _ := preview["topics"].([]any)
if len(topics) != 1 {
t.Fatalf("expected one topic summary, got %#v", topics)
}
})
}
func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 2
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Dispatch 项目", Description: "dispatch"})
if err != nil {
t.Fatalf("create project: %v", err)
}
createWorkflow := func(name string) *domain.WorkflowRecord {
t.Helper()
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: name,
Description: "dispatch test",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow %s: %v", name, err)
}
return workflow
}
workflowA := createWorkflow("并发工作流 A")
workflowB := createWorkflow("并发工作流 B")
runA, err := dataStore.CreateRun(ctx, workflowA.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run A: %v", err)
}
runB, err := dataStore.CreateRun(ctx, workflowB.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run B: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
runs, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatalf("list runs: %v", err)
}
runningCount := 0
for _, run := range runs {
if run.Status == domain.RunStatusRunning {
runningCount++
}
}
if runningCount != 2 {
t.Fatalf("expected 2 running runs after dispatch, got %d", runningCount)
}
waitForRunStatus(t, ctx, dataStore, runA.ID, domain.RunStatusCompleted, 8*time.Second)
waitForRunStatus(t, ctx, dataStore, runB.ID, domain.RunStatusCompleted, 8*time.Second)
}
func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 3
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Override 项目", Description: "override"})
if err != nil {
t.Fatalf("create project: %v", err)
}
limitedWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "受限工作流",
Description: "limited",
Type: domain.WorkflowTypeBatch,
MaxConcurrentRuns: 1,
Nodes: []domain.WorkflowNode{{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create limited workflow: %v", err)
}
freeWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "自由工作流",
Description: "free",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create free workflow: %v", err)
}
runLimitedA, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create limited run A: %v", err)
}
runLimitedB, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create limited run B: %v", err)
}
runFree, err := dataStore.CreateRun(ctx, freeWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create free run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
limitedRunAState, err := dataStore.GetRun(ctx, runLimitedA.ID)
if err != nil {
t.Fatalf("get limited run A: %v", err)
}
limitedRunBState, err := dataStore.GetRun(ctx, runLimitedB.ID)
if err != nil {
t.Fatalf("get limited run B: %v", err)
}
freeRunState, err := dataStore.GetRun(ctx, runFree.ID)
if err != nil {
t.Fatalf("get free run: %v", err)
}
if limitedRunAState.Status != domain.RunStatusRunning {
t.Fatalf("expected first limited run to be running, got %s", limitedRunAState.Status)
}
if limitedRunBState.Status != domain.RunStatusQueued {
t.Fatalf("expected second limited run to remain queued, got %s", limitedRunBState.Status)
}
if freeRunState.Status != domain.RunStatusRunning {
t.Fatalf("expected free workflow run to be running, got %s", freeRunState.Status)
}
waitForRunStatus(t, ctx, dataStore, runLimitedA.ID, domain.RunStatusCompleted, 8*time.Second)
waitForRunStatus(t, ctx, dataStore, runFree.ID, domain.RunStatusCompleted, 8*time.Second)
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available second pass: %v", err)
}
waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second)
}
func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 3
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Parallel DAG 项目", Description: "parallel dag"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "并行分支工作流",
Description: "parallel dag",
Type: domain.WorkflowTypeBatch,
MaxConcurrentRuns: 3,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "OSS 数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "SOURCE_TEXT", Value: "parallel"}},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "branch-a",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "branch a",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-a", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"a\"}' > result-a.json"}},
},
Position: domain.Position{X: 200, Y: -120},
},
{
ID: "branch-b",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "branch b",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-b", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"b\"}' > result-b.json"}},
},
Position: domain.Position{X: 200, Y: 0},
},
{
ID: "branch-c",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "branch c",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-c", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"c\"}' > result-c.json"}},
},
Position: domain.Position{X: 200, Y: 120},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-a", Source: "source", SourceHandle: "out-1", Target: "branch-a", TargetHandle: "in-1"},
{ID: "source-b", Source: "source", SourceHandle: "out-1", Target: "branch-b", TargetHandle: "in-1"},
{ID: "source-c", Source: "source", SourceHandle: "out-1", Target: "branch-c", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
runningRun := waitForRunningNodeCount(t, ctx, dataStore, run.ID, 3, 5*time.Second)
if runningRun.Status != domain.RunStatusRunning {
t.Fatalf("expected run to be running while branches execute, got %s", runningRun.Status)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second)
for _, nodeID := range []string{"branch-a", "branch-b", "branch-c"} {
if execution := mustNodeExecution(t, completed, nodeID); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected node %s to complete, got %s", nodeID, execution.Status)
}
}
}
func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Timeout 项目", Description: "timeout"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "超时工作流",
Description: "timeout",
Type: domain.WorkflowTypeBatch,
RunTimeoutSeconds: 1,
Nodes: []domain.WorkflowNode{
{
ID: "slow-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "slow",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
failed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get failed run: %v", err)
}
if failed == nil || failed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %#v", failed)
}
if !strings.Contains(failed.Error, "超时") {
t.Fatalf("expected timeout error, got %q", failed.Error)
}
execution := mustNodeExecution(t, failed, "slow-node")
if execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed node execution, got %s", execution.Status)
}
storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
if err != nil {
t.Fatalf("get workflow: %v", err)
}
if storedWorkflow == nil || storedWorkflow.Status != domain.WorkflowStatusFailed {
t.Fatalf("expected failed workflow, got %#v", storedWorkflow)
}
}
func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
markerPath := filepath.Join(t.TempDir(), "retry-marker")
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry 项目", Description: "retry"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "自动重试工作流",
Description: "retry",
Type: domain.WorkflowTypeBatch,
MaxRetryAttempts: 1,
Nodes: []domain.WorkflowNode{
{
ID: "flaky-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "flaky",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{
Key: "COMMAND",
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create initial run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("first process once: %v", err)
}
firstAttempt, err := dataStore.GetRun(ctx, initialRun.ID)
if err != nil {
t.Fatalf("get first attempt: %v", err)
}
if firstAttempt == nil || firstAttempt.Status != domain.RunStatusFailed {
t.Fatalf("expected first attempt to fail, got %#v", firstAttempt)
}
if firstAttempt.Attempt != 1 {
t.Fatalf("expected first attempt number to be 1, got %d", firstAttempt.Attempt)
}
runs, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 2 {
t.Fatalf("expected original + retry run, got %d", len(runs))
}
var retryRun domain.RunRecord
foundRetry := false
for _, candidate := range runs {
if candidate.ID == initialRun.ID {
continue
}
retryRun = candidate
foundRetry = true
break
}
if !foundRetry {
t.Fatalf("expected retry run to be created")
}
if retryRun.Trigger != domain.RunTriggerRetry {
t.Fatalf("expected retry trigger, got %s", retryRun.Trigger)
}
if retryRun.Attempt != 2 {
t.Fatalf("expected retry attempt number to be 2, got %d", retryRun.Attempt)
}
if retryRun.ParentRunID != initialRun.ID {
t.Fatalf("expected retry parent run %s, got %s", initialRun.ID, retryRun.ParentRunID)
}
if retryRun.Status != domain.RunStatusQueued {
t.Fatalf("expected retry run to be queued, got %s", retryRun.Status)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("second process once: %v", err)
}
completedRetry, err := dataStore.GetRun(ctx, retryRun.ID)
if err != nil {
t.Fatalf("get retry run: %v", err)
}
if completedRetry == nil || completedRetry.Status != domain.RunStatusCompleted {
t.Fatalf("expected retry run to complete, got %#v", completedRetry)
}
finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
if err != nil {
t.Fatalf("get final workflow: %v", err)
}
if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted {
t.Fatalf("expected workflow to recover and complete, got %#v", finalWorkflow)
}
}
func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Node Timeout 项目", Description: "node timeout"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "节点超时工作流",
Description: "node timeout",
Type: domain.WorkflowTypeBatch,
NodeTimeoutSeconds: 1,
Nodes: []domain.WorkflowNode{
{
ID: "slow-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "slow",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
failed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get failed run: %v", err)
}
if failed == nil || failed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %#v", failed)
}
if !strings.Contains(failed.Error, "节点执行超时") {
t.Fatalf("expected node timeout error, got %q", failed.Error)
}
}
func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
markerPath := filepath.Join(t.TempDir(), "retry-backoff-marker")
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry Backoff 项目", Description: "retry backoff"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "重试延迟工作流",
Description: "retry backoff",
Type: domain.WorkflowTypeBatch,
MaxRetryAttempts: 1,
RetryBackoffSeconds: 2,
Nodes: []domain.WorkflowNode{
{
ID: "flaky-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "flaky",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{
Key: "COMMAND",
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create initial run: %v", err)
}
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("first process once: %v", err)
}
runs, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 2 {
t.Fatalf("expected original + retry run, got %d", len(runs))
}
var retryRun domain.RunRecord
for _, candidate := range runs {
if candidate.ID != initialRun.ID {
retryRun = candidate
break
}
}
if retryRun.AvailableAt == "" {
t.Fatalf("expected retry run to carry availableAt")
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available before backoff expires: %v", err)
}
stillQueued, err := dataStore.GetRun(ctx, retryRun.ID)
if err != nil {
t.Fatalf("get retry run before backoff: %v", err)
}
if stillQueued == nil || stillQueued.Status != domain.RunStatusQueued {
t.Fatalf("expected retry run to remain queued before backoff expires, got %#v", stillQueued)
}
time.Sleep(2200 * time.Millisecond)
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available after backoff expires: %v", err)
}
completedRetry := waitForRunStatus(t, ctx, dataStore, retryRun.ID, domain.RunStatusCompleted, 8*time.Second)
if completedRetry.Attempt != 2 {
t.Fatalf("expected retry attempt number to stay 2, got %d", completedRetry.Attempt)
}
}