forked from longtao.wu/droboticflow
1339 lines
44 KiB
Go
1339 lines
44 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func localRunnerCommand(t *testing.T) string {
|
|
t.Helper()
|
|
path, err := filepath.Abs("../../backend/operator-runner/main.py")
|
|
if err != nil {
|
|
t.Fatalf("resolve operator runner: %v", err)
|
|
}
|
|
return "python3 " + path
|
|
}
|
|
|
|
func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord {
|
|
t.Helper()
|
|
for _, item := range run.NodeExecutions {
|
|
if item.NodeID == nodeID {
|
|
return item
|
|
}
|
|
}
|
|
t.Fatalf("missing node execution for %s", nodeID)
|
|
return domain.NodeExecutionRecord{}
|
|
}
|
|
|
|
func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name string) domain.ExecutionArtifact {
|
|
t.Helper()
|
|
for _, item := range execution.Artifacts {
|
|
if item.Name == name {
|
|
return item
|
|
}
|
|
}
|
|
t.Fatalf("missing artifact %s on node %s", name, execution.NodeID)
|
|
return domain.ExecutionArtifact{}
|
|
}
|
|
|
|
func writeFile(t *testing.T, path string, content string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatalf("write %s: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func createRosbagDB3(t *testing.T, path string) {
|
|
t.Helper()
|
|
command := exec.Command(
|
|
"python3",
|
|
"-c",
|
|
strings.Join([]string{
|
|
"import sqlite3, sys",
|
|
"conn = sqlite3.connect(sys.argv[1])",
|
|
"cur = conn.cursor()",
|
|
"cur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')",
|
|
"cur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')",
|
|
"cur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")",
|
|
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")",
|
|
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")",
|
|
"conn.commit()",
|
|
"conn.close()",
|
|
}, "; "),
|
|
path,
|
|
)
|
|
if output, err := command.CombinedOutput(); err != nil {
|
|
t.Fatalf("create rosbag db3 %s: %v: %s", path, err, string(output))
|
|
}
|
|
}
|
|
|
|
func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
if run != nil && run.Status == expected {
|
|
return run
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
t.Fatalf("timed out waiting for run %s to become %s, got %#v", runID, expected, run)
|
|
return nil
|
|
}
|
|
|
|
func waitForRunningNodeCount(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected int, timeout time.Duration) *domain.RunRecord {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
if run == nil {
|
|
time.Sleep(100 * time.Millisecond)
|
|
continue
|
|
}
|
|
running := 0
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.Status == domain.NodeExecutionStatusRunning {
|
|
running++
|
|
}
|
|
}
|
|
if running == expected {
|
|
return run
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
t.Fatalf("timed out waiting for run %s to have %d running nodes, got %#v", runID, expected, run)
|
|
return nil
|
|
}
|
|
|
|
func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Worker 项目", Description: "worker"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "批处理工作流",
|
|
Description: "complete path",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "OSS 数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Inputs: []domain.PortDefinition{},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-1", Name: "file", Type: domain.PortTypeFile},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "SOURCE_TEXT", Value: "worker-runtime-test"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "sink",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "sink",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{},
|
|
},
|
|
Position: domain.Position{X: 100, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "source", SourceHandle: "out-1", Target: "sink", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed run, got %#v", completed)
|
|
}
|
|
for _, item := range completed.NodeExecutions {
|
|
if item.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected completed node execution, got %s", item.Status)
|
|
}
|
|
}
|
|
if len(completed.Artifacts) == 0 {
|
|
t.Fatalf("expected artifacts")
|
|
}
|
|
if completed.Artifacts[0].Path == "" {
|
|
t.Fatalf("expected real artifact path")
|
|
}
|
|
if len(completed.ResourceSeries) == 0 {
|
|
t.Fatalf("expected resource samples from operator runner")
|
|
}
|
|
}
|
|
|
|
func TestRuntimePausesForAuditAndResumes(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Audit 项目", Description: "audit"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "审核工作流",
|
|
Description: "audit path",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "review",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "人工审核 (Human Review)",
|
|
Category: "HUMAN",
|
|
Description: "review",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "sink",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{},
|
|
},
|
|
Position: domain.Position{X: 100, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "review", SourceHandle: "approved", Target: "upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
paused, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get paused run: %v", err)
|
|
}
|
|
if paused == nil || paused.Status != domain.RunStatusWaitingForAudit {
|
|
t.Fatalf("expected waiting_for_audit run, got %#v", paused)
|
|
}
|
|
|
|
tasks, err := dataStore.ListAuditTasks(ctx, "all")
|
|
if err != nil {
|
|
t.Fatalf("list audit tasks: %v", err)
|
|
}
|
|
var task *domain.AuditTaskRecord
|
|
for _, item := range tasks {
|
|
if item.RunID == run.ID && item.Status == domain.AuditStatusPending {
|
|
copied := item
|
|
task = &copied
|
|
break
|
|
}
|
|
}
|
|
if task == nil {
|
|
t.Fatalf("expected pending audit task")
|
|
}
|
|
|
|
task.Status = domain.AuditStatusApproved
|
|
task.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertAuditTask(ctx, *task); err != nil {
|
|
t.Fatalf("approve audit task: %v", err)
|
|
}
|
|
paused.Status = domain.RunStatusQueued
|
|
paused.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *paused); err != nil {
|
|
t.Fatalf("queue paused run: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("resume process once: %v", err)
|
|
}
|
|
|
|
resumed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get resumed run: %v", err)
|
|
}
|
|
if resumed == nil || resumed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed resumed run, got %#v", resumed)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Route 项目", Description: "route"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "条件分流工作流",
|
|
Description: "route by handle",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-manifest", Name: "manifest", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "COMMAND", Value: `printf '{"items":[{"kind":"keep","value":1},{"kind":"drop","value":2}]}' > routed.json`},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "route",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "条件分流",
|
|
Category: "LOGIC",
|
|
Description: "route",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "input", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "matched", Name: "matched", Type: domain.PortTypeJSON},
|
|
{ID: "unmatched", Name: "unmatched", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "FIELD_PATH", Value: "kind"},
|
|
{Key: "OPERATOR", Value: "eq"},
|
|
{Key: "EXPECTED_JSON", Value: `"keep"`},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 200, Y: 0},
|
|
},
|
|
{
|
|
ID: "keep-upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "keep sink",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 420, Y: -80},
|
|
},
|
|
{
|
|
ID: "drop-upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "drop sink",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 420, Y: 80},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-route", Source: "source", SourceHandle: "out-manifest", Target: "route", TargetHandle: "in-1"},
|
|
{ID: "route-keep", Source: "route", SourceHandle: "matched", Target: "keep-upload", TargetHandle: "in-1"},
|
|
{ID: "route-drop", Source: "route", SourceHandle: "unmatched", Target: "drop-upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed run, got %#v", completed)
|
|
}
|
|
|
|
routeExecution := mustNodeExecution(t, completed, "route")
|
|
matched := mustArtifact(t, routeExecution, "matched.json")
|
|
unmatched := mustArtifact(t, routeExecution, "unmatched.json")
|
|
if matched.PortID != "matched" || unmatched.PortID != "unmatched" {
|
|
t.Fatalf("expected output port IDs on split artifacts, got matched=%q unmatched=%q", matched.PortID, unmatched.PortID)
|
|
}
|
|
|
|
keepUpload := mustNodeExecution(t, completed, "keep-upload")
|
|
dropUpload := mustNodeExecution(t, completed, "drop-upload")
|
|
keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json")
|
|
dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json")
|
|
keepItems, _ := keepSummary.Data["items"].([]any)
|
|
dropItems, _ := dropSummary.Data["items"].([]any)
|
|
if len(keepItems) != 1 || len(dropItems) != 1 {
|
|
t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems))
|
|
}
|
|
|
|
keepPayload, _ := keepItems[0].(map[string]any)
|
|
dropPayload, _ := dropItems[0].(map[string]any)
|
|
keepRows, _ := keepPayload["items"].([]any)
|
|
dropRows, _ := dropPayload["items"].([]any)
|
|
if len(keepRows) != 1 || len(dropRows) != 1 {
|
|
t.Fatalf("expected one row in each routed payload, got keep=%d drop=%d", len(keepRows), len(dropRows))
|
|
}
|
|
if keepRow, _ := keepRows[0].(map[string]any); keepRow != nil {
|
|
if keepRow["kind"] != "keep" {
|
|
t.Fatalf("unexpected keep row: %#v", keepRow)
|
|
}
|
|
}
|
|
if dropRow, _ := dropRows[0].(map[string]any); dropRow != nil {
|
|
if dropRow["kind"] != "drop" {
|
|
t.Fatalf("unexpected drop row: %#v", dropRow)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) {
|
|
t.Run("command previews text and csv outputs", func(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Preview 项目", Description: "preview"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "文件预览工作流",
|
|
Description: "preview files",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "COMMAND", Value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
commandExecution := mustNodeExecution(t, completed, "command")
|
|
report := mustArtifact(t, commandExecution, "report.csv")
|
|
notes := mustArtifact(t, commandExecution, "notes.txt")
|
|
reportPreview, _ := report.Data["preview"].(map[string]any)
|
|
notesPreview, _ := notes.Data["preview"].(map[string]any)
|
|
if reportPreview["kind"] != "table" {
|
|
t.Fatalf("expected table preview for csv artifact, got %#v", reportPreview)
|
|
}
|
|
if notesPreview["kind"] != "text" {
|
|
t.Fatalf("expected text preview for text artifact, got %#v", notesPreview)
|
|
}
|
|
})
|
|
|
|
t.Run("lerobot dataset summary", func(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
datasetRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(datasetRoot, "meta", "info.json"), `{"robot_type":"aloha","fps":30}`)
|
|
writeFile(t, filepath.Join(datasetRoot, "meta", "stats.json"), `{"episodes":2}`)
|
|
writeFile(t, filepath.Join(datasetRoot, "meta", "tasks.jsonl"), "{\"task\":\"pick\"}\n{\"task\":\"place\"}\n")
|
|
writeFile(t, filepath.Join(datasetRoot, "data", "episode_000.parquet"), "PAR1")
|
|
writeFile(t, filepath.Join(datasetRoot, "videos", "episode_000.mp4"), "video")
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "LeRobot 项目", Description: "lerobot"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "LeRobot 概览",
|
|
Description: "lerobot summary",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: datasetRoot}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "summary",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "LeRobot 数据集概览",
|
|
Category: "ROBOTICS",
|
|
Description: "lerobot",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 240, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json")
|
|
preview, _ := summary.Data["preview"].(map[string]any)
|
|
if preview["format"] != "lerobot" {
|
|
t.Fatalf("expected lerobot preview, got %#v", preview)
|
|
}
|
|
if summary.Data["datasetType"] != "lerobot" {
|
|
t.Fatalf("expected datasetType lerobot, got %#v", summary.Data)
|
|
}
|
|
})
|
|
|
|
t.Run("rosbag sqlite summary", func(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
datasetRoot := t.TempDir()
|
|
dbPath := filepath.Join(datasetRoot, "bag_0.db3")
|
|
createRosbagDB3(t, dbPath)
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "ROS 项目", Description: "rosbag"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "ROSBag 概览",
|
|
Description: "rosbag summary",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "summary",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "ROSBag 数据概览",
|
|
Category: "ROBOTICS",
|
|
Description: "rosbag",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}},
|
|
},
|
|
Position: domain.Position{X: 240, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json")
|
|
preview, _ := summary.Data["preview"].(map[string]any)
|
|
if preview["format"] != "rosbag2" {
|
|
t.Fatalf("expected rosbag2 preview, got %#v", preview)
|
|
}
|
|
topics, _ := preview["topics"].([]any)
|
|
if len(topics) != 1 {
|
|
t.Fatalf("expected one topic summary, got %#v", topics)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 2
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Dispatch 项目", Description: "dispatch"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
createWorkflow := func(name string) *domain.WorkflowRecord {
|
|
t.Helper()
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: name,
|
|
Description: "dispatch test",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow %s: %v", name, err)
|
|
}
|
|
return workflow
|
|
}
|
|
|
|
workflowA := createWorkflow("并发工作流 A")
|
|
workflowB := createWorkflow("并发工作流 B")
|
|
runA, err := dataStore.CreateRun(ctx, workflowA.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run A: %v", err)
|
|
}
|
|
runB, err := dataStore.CreateRun(ctx, workflowB.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run B: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
runs, err := dataStore.ListRuns(ctx, "")
|
|
if err != nil {
|
|
t.Fatalf("list runs: %v", err)
|
|
}
|
|
runningCount := 0
|
|
for _, run := range runs {
|
|
if run.Status == domain.RunStatusRunning {
|
|
runningCount++
|
|
}
|
|
}
|
|
if runningCount != 2 {
|
|
t.Fatalf("expected 2 running runs after dispatch, got %d", runningCount)
|
|
}
|
|
|
|
waitForRunStatus(t, ctx, dataStore, runA.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
waitForRunStatus(t, ctx, dataStore, runB.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 3
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Override 项目", Description: "override"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
limitedWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "受限工作流",
|
|
Description: "limited",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxConcurrentRuns: 1,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create limited workflow: %v", err)
|
|
}
|
|
freeWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "自由工作流",
|
|
Description: "free",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create free workflow: %v", err)
|
|
}
|
|
|
|
runLimitedA, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create limited run A: %v", err)
|
|
}
|
|
runLimitedB, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create limited run B: %v", err)
|
|
}
|
|
runFree, err := dataStore.CreateRun(ctx, freeWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create free run: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
limitedRunAState, err := dataStore.GetRun(ctx, runLimitedA.ID)
|
|
if err != nil {
|
|
t.Fatalf("get limited run A: %v", err)
|
|
}
|
|
limitedRunBState, err := dataStore.GetRun(ctx, runLimitedB.ID)
|
|
if err != nil {
|
|
t.Fatalf("get limited run B: %v", err)
|
|
}
|
|
freeRunState, err := dataStore.GetRun(ctx, runFree.ID)
|
|
if err != nil {
|
|
t.Fatalf("get free run: %v", err)
|
|
}
|
|
if limitedRunAState.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected first limited run to be running, got %s", limitedRunAState.Status)
|
|
}
|
|
if limitedRunBState.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected second limited run to remain queued, got %s", limitedRunBState.Status)
|
|
}
|
|
if freeRunState.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected free workflow run to be running, got %s", freeRunState.Status)
|
|
}
|
|
|
|
waitForRunStatus(t, ctx, dataStore, runLimitedA.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
waitForRunStatus(t, ctx, dataStore, runFree.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available second pass: %v", err)
|
|
}
|
|
waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
}
|
|
|
|
func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 3
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Parallel DAG 项目", Description: "parallel dag"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "并行分支工作流",
|
|
Description: "parallel dag",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxConcurrentRuns: 3,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "OSS 数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "SOURCE_TEXT", Value: "parallel"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "branch-a",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "branch a",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-a", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"a\"}' > result-a.json"}},
|
|
},
|
|
Position: domain.Position{X: 200, Y: -120},
|
|
},
|
|
{
|
|
ID: "branch-b",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "branch b",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-b", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"b\"}' > result-b.json"}},
|
|
},
|
|
Position: domain.Position{X: 200, Y: 0},
|
|
},
|
|
{
|
|
ID: "branch-c",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "branch c",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-c", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"c\"}' > result-c.json"}},
|
|
},
|
|
Position: domain.Position{X: 200, Y: 120},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-a", Source: "source", SourceHandle: "out-1", Target: "branch-a", TargetHandle: "in-1"},
|
|
{ID: "source-b", Source: "source", SourceHandle: "out-1", Target: "branch-b", TargetHandle: "in-1"},
|
|
{ID: "source-c", Source: "source", SourceHandle: "out-1", Target: "branch-c", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
runningRun := waitForRunningNodeCount(t, ctx, dataStore, run.ID, 3, 5*time.Second)
|
|
if runningRun.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected run to be running while branches execute, got %s", runningRun.Status)
|
|
}
|
|
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second)
|
|
for _, nodeID := range []string{"branch-a", "branch-b", "branch-c"} {
|
|
if execution := mustNodeExecution(t, completed, nodeID); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected node %s to complete, got %s", nodeID, execution.Status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Timeout 项目", Description: "timeout"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "超时工作流",
|
|
Description: "timeout",
|
|
Type: domain.WorkflowTypeBatch,
|
|
RunTimeoutSeconds: 1,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "slow-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "slow",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
failed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get failed run: %v", err)
|
|
}
|
|
if failed == nil || failed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %#v", failed)
|
|
}
|
|
if !strings.Contains(failed.Error, "超时") {
|
|
t.Fatalf("expected timeout error, got %q", failed.Error)
|
|
}
|
|
execution := mustNodeExecution(t, failed, "slow-node")
|
|
if execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed node execution, got %s", execution.Status)
|
|
}
|
|
|
|
storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
|
|
if err != nil {
|
|
t.Fatalf("get workflow: %v", err)
|
|
}
|
|
if storedWorkflow == nil || storedWorkflow.Status != domain.WorkflowStatusFailed {
|
|
t.Fatalf("expected failed workflow, got %#v", storedWorkflow)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
markerPath := filepath.Join(t.TempDir(), "retry-marker")
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry 项目", Description: "retry"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "自动重试工作流",
|
|
Description: "retry",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxRetryAttempts: 1,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "flaky-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "flaky",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{
|
|
Key: "COMMAND",
|
|
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
|
|
}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create initial run: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("first process once: %v", err)
|
|
}
|
|
|
|
firstAttempt, err := dataStore.GetRun(ctx, initialRun.ID)
|
|
if err != nil {
|
|
t.Fatalf("get first attempt: %v", err)
|
|
}
|
|
if firstAttempt == nil || firstAttempt.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected first attempt to fail, got %#v", firstAttempt)
|
|
}
|
|
if firstAttempt.Attempt != 1 {
|
|
t.Fatalf("expected first attempt number to be 1, got %d", firstAttempt.Attempt)
|
|
}
|
|
|
|
runs, err := dataStore.ListRuns(ctx, "")
|
|
if err != nil {
|
|
t.Fatalf("list runs: %v", err)
|
|
}
|
|
if len(runs) != 2 {
|
|
t.Fatalf("expected original + retry run, got %d", len(runs))
|
|
}
|
|
|
|
var retryRun domain.RunRecord
|
|
foundRetry := false
|
|
for _, candidate := range runs {
|
|
if candidate.ID == initialRun.ID {
|
|
continue
|
|
}
|
|
retryRun = candidate
|
|
foundRetry = true
|
|
break
|
|
}
|
|
if !foundRetry {
|
|
t.Fatalf("expected retry run to be created")
|
|
}
|
|
if retryRun.Trigger != domain.RunTriggerRetry {
|
|
t.Fatalf("expected retry trigger, got %s", retryRun.Trigger)
|
|
}
|
|
if retryRun.Attempt != 2 {
|
|
t.Fatalf("expected retry attempt number to be 2, got %d", retryRun.Attempt)
|
|
}
|
|
if retryRun.ParentRunID != initialRun.ID {
|
|
t.Fatalf("expected retry parent run %s, got %s", initialRun.ID, retryRun.ParentRunID)
|
|
}
|
|
if retryRun.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected retry run to be queued, got %s", retryRun.Status)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("second process once: %v", err)
|
|
}
|
|
|
|
completedRetry, err := dataStore.GetRun(ctx, retryRun.ID)
|
|
if err != nil {
|
|
t.Fatalf("get retry run: %v", err)
|
|
}
|
|
if completedRetry == nil || completedRetry.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected retry run to complete, got %#v", completedRetry)
|
|
}
|
|
|
|
finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
|
|
if err != nil {
|
|
t.Fatalf("get final workflow: %v", err)
|
|
}
|
|
if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted {
|
|
t.Fatalf("expected workflow to recover and complete, got %#v", finalWorkflow)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Node Timeout 项目", Description: "node timeout"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "节点超时工作流",
|
|
Description: "node timeout",
|
|
Type: domain.WorkflowTypeBatch,
|
|
NodeTimeoutSeconds: 1,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "slow-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "slow",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
failed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get failed run: %v", err)
|
|
}
|
|
if failed == nil || failed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %#v", failed)
|
|
}
|
|
if !strings.Contains(failed.Error, "节点执行超时") {
|
|
t.Fatalf("expected node timeout error, got %q", failed.Error)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
markerPath := filepath.Join(t.TempDir(), "retry-backoff-marker")
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry Backoff 项目", Description: "retry backoff"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "重试延迟工作流",
|
|
Description: "retry backoff",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxRetryAttempts: 1,
|
|
RetryBackoffSeconds: 2,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "flaky-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "flaky",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{
|
|
Key: "COMMAND",
|
|
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
|
|
}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create initial run: %v", err)
|
|
}
|
|
|
|
runtime, err := NewRuntime(dataStore, "local", "droboticflow", "droboticflow-registry", "operator-runner:latest", localRunnerCommand(t))
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("first process once: %v", err)
|
|
}
|
|
|
|
runs, err := dataStore.ListRuns(ctx, "")
|
|
if err != nil {
|
|
t.Fatalf("list runs: %v", err)
|
|
}
|
|
if len(runs) != 2 {
|
|
t.Fatalf("expected original + retry run, got %d", len(runs))
|
|
}
|
|
var retryRun domain.RunRecord
|
|
for _, candidate := range runs {
|
|
if candidate.ID != initialRun.ID {
|
|
retryRun = candidate
|
|
break
|
|
}
|
|
}
|
|
if retryRun.AvailableAt == "" {
|
|
t.Fatalf("expected retry run to carry availableAt")
|
|
}
|
|
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available before backoff expires: %v", err)
|
|
}
|
|
stillQueued, err := dataStore.GetRun(ctx, retryRun.ID)
|
|
if err != nil {
|
|
t.Fatalf("get retry run before backoff: %v", err)
|
|
}
|
|
if stillQueued == nil || stillQueued.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected retry run to remain queued before backoff expires, got %#v", stillQueued)
|
|
}
|
|
|
|
time.Sleep(2200 * time.Millisecond)
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available after backoff expires: %v", err)
|
|
}
|
|
completedRetry := waitForRunStatus(t, ctx, dataStore, retryRun.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if completedRetry.Attempt != 2 {
|
|
t.Fatalf("expected retry attempt number to stay 2, got %d", completedRetry.Attempt)
|
|
}
|
|
}
|