package worker import ( "context" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "sort" "strings" "testing" "time" "droboticflow/internal/domain" "droboticflow/internal/k8s" "droboticflow/internal/store" ) type testLauncher struct { launch func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) inspect func(context.Context, string, string) (*k8s.JobInspection, error) gpuCapacity func(context.Context) (*k8s.GPUCapacitySnapshot, error) terminateRun func(context.Context, string, string) error samplePodGPUForRun func(context.Context, string, string) (k8s.GPUSample, error) tailNodePodLogs func(context.Context, string, string, int64) (string, error) } func (launcher testLauncher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) { if launcher.tailNodePodLogs == nil { return "", fmt.Errorf("no pod logs for run %s node %s", runID, nodeID) } return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines) } func (launcher testLauncher) Launch(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { if launcher.launch == nil { return nil, fmt.Errorf("test launcher launch not configured") } return launcher.launch(ctx, input) } func (launcher testLauncher) Inspect(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) { if launcher.inspect == nil { return &k8s.JobInspection{State: k8s.JobStateMissing}, nil } return launcher.inspect(ctx, runID, nodeID) } func (launcher testLauncher) TerminateRun(ctx context.Context, namespace string, runID string) error { if launcher.terminateRun == nil { return nil } return launcher.terminateRun(ctx, namespace, runID) } func (launcher testLauncher) SamplePodGPUForRun(ctx context.Context, runID string, nodeID string) (k8s.GPUSample, error) { if launcher.samplePodGPUForRun == nil { return k8s.GPUSample{}, fmt.Errorf("no running pod for run %s node %s", runID, nodeID) } return launcher.samplePodGPUForRun(ctx, runID, nodeID) } func (launcher testLauncher) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) { if launcher.gpuCapacity == nil { return &k8s.GPUCapacitySnapshot{ResourceName: "nvidia.com/gpu", SelectorSummary: ""}, nil } return launcher.gpuCapacity(ctx) } func operatorRunnerPath(t *testing.T) string { t.Helper() path, err := filepath.Abs("../../backend/operator-runner/main.py") if err != nil { t.Fatalf("resolve operator runner: %v", err) } return path } func newTestRuntime(t *testing.T, dataStore store.Store) (*Runtime, error) { t.Helper() t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "true") runnerPath := operatorRunnerPath(t) return &Runtime{ store: dataStore, operatorRunnerImage: "operator-runner:latest", workspaceRoot: t.TempDir(), launcher: testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { env := append([]string{}, os.Environ()...) keys := make([]string, 0, len(input.Env)) for key := range input.Env { keys = append(keys, key) } sort.Strings(keys) env = append(env, "RUN_ID="+input.RunID, "WORKFLOW_ID="+input.WorkflowID, "NODE_ID="+input.NodeID, "NODE_LABEL="+input.NodeLabel, ) for _, key := range keys { env = append(env, key+"="+input.Env[key]) } cmd := exec.CommandContext(ctx, "python3", runnerPath) cmd.Env = env output, err := cmd.CombinedOutput() if err != nil { return nil, fmt.Errorf("%w: %s", err, string(output)) } return &k8s.LaunchResult{ Reference: &k8s.JobReference{ JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID), Namespace: "droboticflow", }, Output: string(output), }, nil }, }, }, nil } func newTestRuntimeWithLauncher(t *testing.T, dataStore store.Store, launcher jobLauncher) *Runtime { t.Helper() t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "true") return &Runtime{ store: dataStore, operatorRunnerImage: "operator-runner:latest", workspaceRoot: t.TempDir(), launcher: launcher, } } func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord { t.Helper() for _, item := range run.NodeExecutions { if item.NodeID == nodeID { return item } } t.Fatalf("missing node execution for %s", nodeID) return domain.NodeExecutionRecord{} } func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name string) domain.ExecutionArtifact { t.Helper() for _, item := range execution.Artifacts { if item.Name == name { return item } } t.Fatalf("missing artifact %s on node %s", name, execution.NodeID) return domain.ExecutionArtifact{} } func mustArtifactPayload(t *testing.T, artifact domain.ExecutionArtifact) map[string]any { t.Helper() if artifact.Data != nil { if dataRef, ok := artifact.Data["dataRef"].(map[string]any); ok { if payloadPath, ok := dataRef["payloadPath"].(string); ok && strings.TrimSpace(payloadPath) != "" { payload, err := os.ReadFile(payloadPath) if err != nil { t.Fatalf("read artifact payload %s: %v", payloadPath, err) } var decoded map[string]any if err := json.Unmarshal(payload, &decoded); err != nil { t.Fatalf("decode artifact payload %s: %v", payloadPath, err) } return decoded } } return artifact.Data } if strings.TrimSpace(artifact.Path) != "" { payload, err := os.ReadFile(artifact.Path) if err == nil { var decoded map[string]any if err := json.Unmarshal(payload, &decoded); err == nil { return decoded } } } t.Fatalf("artifact %s has no structured payload", artifact.Name) return nil } func mustProcessOnceRun(t *testing.T, runtime *Runtime, ctx context.Context) *domain.RunRecord { t.Helper() result, err := runtime.ProcessOnce(ctx) if err != nil { t.Fatalf("process run: %v", err) } if result == nil || result.Run == nil { t.Fatalf("expected process-once to return a run, got %#v", result) } return result.Run } func TestProcessOnceResultJSONUsesLightweightRunSummary(t *testing.T) { run := &domain.RunRecord{ ID: "run-1", WorkflowID: "workflow-1", WorkflowName: "workflow", ProjectID: "project-1", ProjectName: "project", Status: domain.RunStatusCompleted, Attempt: 2, CurrentNodeID: "node-2", ExecutionLeaseOwner: "worker-1", ExecutionLeaseExpiresAt: "2026-07-02T00:00:00Z", UpdatedAt: "2026-07-02T00:00:01Z", Logs: []domain.LogEntryRecord{{ Level: "INFO", Message: strings.Repeat("log-", 100), }}, NodeExecutions: []domain.NodeExecutionRecord{{ NodeID: "node-1", Status: domain.NodeExecutionStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ ID: "artifact-1", Name: "large.json", Data: map[string]any{"payload": strings.Repeat("data-", 100)}, }}, Logs: []domain.LogEntryRecord{{Message: strings.Repeat("node-log-", 100)}}, }}, Artifacts: []domain.ExecutionArtifact{{ ID: "run-artifact-1", Name: "report.json", Data: map[string]any{"payload": strings.Repeat("report-", 100)}, }}, } result := newProcessOnceResult(ProcessOnceStatusClaimed, run) if result.Run != run { t.Fatalf("expected runtime result to retain the full run pointer") } encoded, err := json.Marshal(result) if err != nil { t.Fatalf("marshal process-once result: %v", err) } body := string(encoded) for _, forbidden := range []string{"nodeExecutions", "logs", "resourceSeries", "k8sRefs", "large.json", "node-log-", "report-"} { if strings.Contains(body, forbidden) { t.Fatalf("process-once response leaked full run field %q in %s", forbidden, body) } } for _, expected := range []string{`"status":"claimed"`, `"id":"run-1"`, `"nodeExecutionCount":1`, `"artifactCount":2`, `"currentNodeId":"node-2"`} { if !strings.Contains(body, expected) { t.Fatalf("process-once response missing %s in %s", expected, body) } } } func TestRuntimeProcessAvailableSkipsAlreadyActiveRun(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "active run", Description: "skip"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "workflow", Description: "skip active run", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard, Nodes: []domain.WorkflowNode{{ ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "source", Outputs: []domain.PortDefinition{{ ID: "out-1", Name: "payload", Type: domain.PortTypeJSON, }}, }, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } launchCount := 0 runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { launchCount++ return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: "job-" + input.RunID + "-" + input.NodeID, Namespace: "droboticflow"}, Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, nil }, }) if !runtime.tryMarkRunActive(run.ID) { t.Fatal("expected active run mark to succeed") } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } if launchCount != 0 { t.Fatalf("expected active run to be skipped, got %d launches", launchCount) } } // TestProcessOnceSkippedLeaseActiveReleasesLeaseAndKeepsRunning guards the race // where ProcessOnce's ClaimNextQueuedRun wins the mongo claim but tryMarkRunActive // fails because another goroutine in this worker already owns the run. // // The old behavior reverted status to Queued while leaving currentNodeId set and // leaving the just-claimed lease held — an inconsistent "queued+currentNodeId" // state no other queued-writer emits (and the one that matched a phantom symptom // we chased). It also blocked ProcessAvailable's claim filter for the lease TTL // (~15min), orphaning the run. The fix releases the spurious lease and leaves // status=Running (the active processor owns the run). func TestProcessOnceSkippedLeaseActiveReleasesLeaseAndKeepsRunning(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "lease race", Description: "skip"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "workflow", Description: "lease race", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard, Nodes: []domain.WorkflowNode{{ ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "source"}, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } launchCount := 0 runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { launchCount++ return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: "job-" + input.RunID + "-" + input.NodeID, Namespace: "droboticflow"}, Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, nil }, }) // Pre-mark the run active so ProcessOnce's tryMarkRunActive fails, simulating // the race where another goroutine already owns the in-memory activeRuns slot. if !runtime.tryMarkRunActive(run.ID) { t.Fatal("expected active run mark to succeed") } result, err := runtime.ProcessOnce(ctx) if err != nil { t.Fatalf("process once: %v", err) } if result == nil || result.Status != ProcessOnceStatusSkippedLeaseActive { t.Fatalf("expected skipped_lease_active, got %#v", result) } if launchCount != 0 { t.Fatalf("expected no launch (run already active), got %d", launchCount) } stored, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if stored.Status != domain.RunStatusRunning { t.Fatalf("expected run to stay Running after skipped claim (active processor owns it), got %s", stored.Status) } if strings.TrimSpace(stored.ExecutionLeaseOwner) != "" { t.Fatalf("expected spurious lease released so it does not block the next claim for the lease TTL, got owner=%q", stored.ExecutionLeaseOwner) } } func TestExecuteNodeFallsBackToOperatorResourceDefaults(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() operator, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ Name: "OpenVLA 微调训练", Type: "builtin", Category: "AI", Description: "gpu operator", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "model", Type: domain.PortTypeFile}}, Resources: &domain.OperatorResources{ CPURequest: "8", CPULimit: "24", MemoryRequest: "32Gi", MemoryLimit: "128Gi", GPULimit: "1", SharedMemorySize: "12Gi", }, }) if err != nil { t.Fatalf("create operator: %v", err) } project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "GPU 项目", Description: "resources"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "GPU 工作流", Description: "operator resource defaults", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard, Nodes: []domain.WorkflowNode{{ ID: "train", Type: "operator", Data: domain.WorkflowNodeData{ Label: "OpenVLA 微调训练", OperatorID: operator.ID, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "model", Type: domain.PortTypeFile}}, }, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } var captured k8s.LaunchInput launchCount := 0 runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { launchCount++ captured = input if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" { if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o600); err != nil { return nil, err } } return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: "job-" + input.NodeID, Namespace: "droboticflow"}, Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, nil }, }) if _, _, err := runtime.executeNode(ctx, run, workflow.Nodes[0], nil, nil); err != nil { t.Fatalf("execute node: %v", err) } if launchCount != 1 { t.Fatalf("expected one launch, got %d", launchCount) } if captured.GPULimit != "1" || captured.Env["GPU_LIMIT"] != "1" { t.Fatalf("expected operator GPU default to reach launcher/env, got gpu=%q env=%q", captured.GPULimit, captured.Env["GPU_LIMIT"]) } if captured.CPURequest != "8" || captured.CPULimit != "24" || captured.MemoryRequest != "32Gi" || captured.MemoryLimit != "128Gi" { t.Fatalf("expected operator resource defaults, got %#v", captured) } if captured.SharedMemorySize != "12Gi" { t.Fatalf("expected operator shared memory default, got %q", captured.SharedMemorySize) } } func TestResolveEffectiveNodeSharedMemoryPrecedence(t *testing.T) { operatorResources := &domain.OperatorResources{SharedMemorySize: "8Gi"} operatorDefault := resolveEffectiveNodeResources(domain.WorkflowNode{}, nil, operatorResources) if operatorDefault.SharedMemorySize != "8Gi" { t.Fatalf("operator shared memory default = %q, want 8Gi", operatorDefault.SharedMemorySize) } nodeOverride := resolveEffectiveNodeResources(domain.WorkflowNode{ Data: domain.WorkflowNodeData{SharedMemorySize: "16Gi"}, }, nil, operatorResources) if nodeOverride.SharedMemorySize != "16Gi" { t.Fatalf("node shared memory override = %q, want 16Gi", nodeOverride.SharedMemorySize) } platformFallback := resolveEffectiveNodeResources(domain.WorkflowNode{}, nil, nil) if platformFallback.SharedMemorySize != "" { t.Fatalf("empty shared memory should be left for launcher platform fallback, got %q", platformFallback.SharedMemorySize) } } func TestProcessRunFailsInvalidWorkflowWithDuplicateInputEdges(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ Name: "非法工作流项目", Description: "duplicate input edges", }) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "非法工作流", Description: "duplicate input edges", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source-a", Type: "operator", Data: domain.WorkflowNodeData{ Label: "上游 A", Category: "IO", Description: "source a", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, }, }, { ID: "source-b", Type: "operator", Data: domain.WorkflowNodeData{ Label: "上游 B", Category: "IO", Description: "source b", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, }, }, { ID: "upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", Category: "IO", Description: "upload", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "result", Type: domain.PortTypeJSON}}, }, }, }, Edges: []domain.WorkflowEdge{ {ID: "edge-a", Source: "source-a", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, {ID: "edge-b", Source: "source-b", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } processed, err := runtime.processRun(ctx, run) if err != nil { t.Fatalf("process run: %v", err) } if processed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %s", processed.Status) } if !strings.Contains(processed.Error, "同一输入端口存在多条上游连线") { t.Fatalf("expected duplicate input edge error, got %q", processed.Error) } if len(processed.NodeExecutions) != 0 { t.Fatalf("expected no node executions for invalid workflow, got %#v", processed.NodeExecutions) } } func TestProcessRunFailsInvalidIncrementalDirectoryDepth(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ Name: "非法目录深度项目", Description: "invalid incremental depth", }) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "非法目录深度工作流", Description: "invalid incremental depth", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "INCREMENTAL"}, {Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"}, {Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "zero"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } processed := mustProcessOnceRun(t, runtime, ctx) if processed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %s", processed.Status) } if !strings.Contains(processed.Error, "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") { t.Fatalf("expected directory depth error, got %q", processed.Error) } execution := mustNodeExecution(t, processed, "source") if execution.Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected failed source execution, got %s", execution.Status) } if !strings.Contains(execution.Error, "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") { t.Fatalf("expected node directory depth error, got %q", execution.Error) } } func TestProcessRunFailsInvalidIncrementalUnit(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ Name: "非法增量单位项目", Description: "invalid incremental unit", }) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "非法增量单位工作流", Description: "invalid incremental unit", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "INCREMENTAL"}, {Key: "INCREMENTAL_UNIT", Value: "ROW"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } processed := mustProcessOnceRun(t, runtime, ctx) if processed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %s", processed.Status) } if !strings.Contains(processed.Error, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY") { t.Fatalf("expected incremental unit error, got %q", processed.Error) } execution := mustNodeExecution(t, processed, "source") if execution.Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected failed source execution, got %s", execution.Status) } if !strings.Contains(execution.Error, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY") { t.Fatalf("expected node incremental unit error, got %q", execution.Error) } } func TestProcessRunFailsInvalidScanMode(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ Name: "非法扫描模式项目", Description: "invalid scan mode", }) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "非法扫描模式工作流", Description: "invalid scan mode", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "ROW"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } processed := mustProcessOnceRun(t, runtime, ctx) if processed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %s", processed.Status) } if !strings.Contains(processed.Error, "SCAN_MODE must be FULL or INCREMENTAL") { t.Fatalf("expected scan mode error, got %q", processed.Error) } execution := mustNodeExecution(t, processed, "source") if execution.Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected failed source execution, got %s", execution.Status) } if !strings.Contains(execution.Error, "SCAN_MODE must be FULL or INCREMENTAL") { t.Fatalf("expected node scan mode error, got %q", execution.Error) } } func TestProcessRunFailsInvalidDataSource(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ Name: "非法数据源项目", Description: "invalid data source", }) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "非法数据源工作流", Description: "invalid data source", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "ROW"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "INCREMENTAL"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } processed := mustProcessOnceRun(t, runtime, ctx) if processed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %s", processed.Status) } if !strings.Contains(processed.Error, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") { t.Fatalf("expected data source error, got %q", processed.Error) } execution := mustNodeExecution(t, processed, "source") if execution.Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected failed source execution, got %s", execution.Status) } if !strings.Contains(execution.Error, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") { t.Fatalf("expected node data source error, got %q", execution.Error) } } func TestSourceStateFromCheckpointPrefersCheckpointFilePayload(t *testing.T) { workflow := &domain.WorkflowRecord{ID: "workflow-1"} run := &domain.RunRecord{ID: "run-1"} checkpointPath := filepath.Join(t.TempDir(), "source-checkpoint.json") writeFile(t, checkpointPath, `{ "sourceFingerprint": "fp-1", "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test", "bucket": "datastorage", "prefix": "algo/X-platform-test", "pattern": "", "scanMode": "INCREMENTAL", "directoryDepth": 1, "seenFingerprints": ["a", "b"], "listedCount": 28, "newCount": 0, "scanStartedAt": "2026-04-12T15:45:42Z", "scanCompletedAt": "2026-04-12T15:45:55Z" }`) execution := domain.NodeExecutionRecord{ NodeID: "scan-source", Artifacts: []domain.ExecutionArtifact{ { Name: "source-checkpoint.json", Path: checkpointPath, Data: map[string]any{ "sourceFingerprint": "fp-1", "listedCount": 28.0, "newCount": 28.0, }, }, }, } state, err := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0]) if err != nil { t.Fatalf("source state from checkpoint: %v", err) } if state == nil { t.Fatalf("expected source state") } if state.LastNewCount != 0 { t.Fatalf("expected checkpoint file payload to win, got lastNewCount=%d", state.LastNewCount) } if state.LastListedCount != 28 { t.Fatalf("expected lastListedCount=28, got %d", state.LastListedCount) } if state.LastRunID != run.ID { t.Fatalf("expected lastRunID=%s, got %s", run.ID, state.LastRunID) } } func TestSourceStateFromCheckpointRejectsUnreadableOrInvalidCheckpointFiles(t *testing.T) { workflow := &domain.WorkflowRecord{ID: "workflow-1"} run := &domain.RunRecord{ID: "run-1"} execution := domain.NodeExecutionRecord{NodeID: "scan-source"} missingPath := filepath.Join(t.TempDir(), "missing-source-checkpoint.json") _, err := sourceStateFromCheckpoint(workflow, run, execution, &domain.ExecutionArtifact{ Name: "source-checkpoint.json", Path: missingPath, Data: map[string]any{ "sourceFingerprint": "stale-inline", "listedCount": 1, "newCount": 1, }, }) if err == nil || !strings.Contains(err.Error(), "read source checkpoint") { t.Fatalf("expected unreadable checkpoint error, got %v", err) } invalidPath := filepath.Join(t.TempDir(), "source-checkpoint.json") writeFile(t, invalidPath, `{`) _, err = sourceStateFromCheckpoint(workflow, run, execution, &domain.ExecutionArtifact{ Name: "source-checkpoint.json", Path: invalidPath, Data: map[string]any{ "sourceFingerprint": "stale-inline", "listedCount": 1, "newCount": 1, }, }) if err == nil || !strings.Contains(err.Error(), "parse source checkpoint") { t.Fatalf("expected invalid checkpoint JSON error, got %v", err) } } func TestSourceStateFromCheckpointRejectsLossyCheckpointCounts(t *testing.T) { workflow := &domain.WorkflowRecord{ID: "workflow-1"} run := &domain.RunRecord{ID: "run-1"} execution := domain.NodeExecutionRecord{ NodeID: "scan-source", Artifacts: []domain.ExecutionArtifact{{ Name: "source-checkpoint.json", Data: map[string]any{ "sourceFingerprint": "fp-lossy", "directoryDepth": float64(1), "listedCount": 28.5, "newCount": 0.0, }, }}, } _, err := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0]) if err == nil || !strings.Contains(err.Error(), "listedCount must be an integer") { t.Fatalf("expected lossy listedCount error, got %v", err) } } func TestCommitSourceStatesPrefersPersistedRunSnapshot(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } workflow := &domain.WorkflowRecord{ID: "workflow-1"} persistedRun := domain.RunRecord{ ID: "run-1", WorkflowID: workflow.ID, NodeExecutions: []domain.NodeExecutionRecord{{ NodeID: "scan-source", Artifacts: []domain.ExecutionArtifact{{ Name: "source-checkpoint.json", Data: map[string]any{ "sourceFingerprint": "fp-1", "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test", "bucket": "datastorage", "prefix": "algo/X-platform-test", "scanMode": "INCREMENTAL", "directoryDepth": 1.0, "seenFingerprints": []any{"a", "b"}, "listedCount": 28.0, "newCount": 0.0, "scanStartedAt": "2026-04-12T15:45:42Z", "scanCompletedAt": "2026-04-12T15:45:55Z", }, }}, }}, } if err := dataStore.UpsertRun(ctx, persistedRun); err != nil { t.Fatalf("upsert persisted run: %v", err) } inMemoryRun := &domain.RunRecord{ ID: "run-1", WorkflowID: workflow.ID, NodeExecutions: []domain.NodeExecutionRecord{{ NodeID: "scan-source", Artifacts: []domain.ExecutionArtifact{{ Name: "source-checkpoint.json", Data: map[string]any{ "sourceFingerprint": "fp-1", "directoryDepth": 1.0, "listedCount": 28.0, "newCount": 28.0, }, }}, }}, } if err := runtime.commitSourceStates(ctx, workflow, inMemoryRun); err != nil { t.Fatalf("commit source states: %v", err) } state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-1") if err != nil { t.Fatalf("get source state: %v", err) } if state == nil { t.Fatalf("expected committed source state") } if state.LastNewCount != 0 { t.Fatalf("expected committed source state to use persisted run snapshot, got %d", state.LastNewCount) } } func TestCommitSourceStatesPreservesSeenFingerprintsFromTypedSlice(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } workflow := &domain.WorkflowRecord{ID: "workflow-typed-slice"} persistedRun := domain.RunRecord{ ID: "run-typed-slice", WorkflowID: workflow.ID, NodeExecutions: []domain.NodeExecutionRecord{{ NodeID: "scan-source", Artifacts: []domain.ExecutionArtifact{{ Name: "source-checkpoint.json", Data: map[string]any{ "sourceFingerprint": "fp-typed-slice", "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test/raw", "bucket": "datastorage", "prefix": "algo/X-platform-test/raw", "scanMode": "INCREMENTAL", "directoryDepth": 1.0, "seenFingerprints": []string{"a", "b", "c"}, "listedCount": 4.0, "newCount": 0.0, }, }}, }}, } if err := dataStore.UpsertRun(ctx, persistedRun); err != nil { t.Fatalf("upsert persisted run: %v", err) } if err := runtime.commitSourceStates(ctx, workflow, &persistedRun); err != nil { t.Fatalf("commit source states: %v", err) } state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-typed-slice") if err != nil { t.Fatalf("get source state: %v", err) } if state == nil { t.Fatalf("expected committed source state") } if len(state.SeenFingerprints) != 3 { t.Fatalf("expected 3 seen fingerprints, got %#v", state.SeenFingerprints) } if strings.Join(state.SeenFingerprints, ",") != "a,b,c" { t.Fatalf("unexpected seen fingerprints %#v", state.SeenFingerprints) } } func writeFile(t *testing.T, path string, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) } if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatalf("write %s: %v", path, err) } } func createRosbagDB3(t *testing.T, path string) { t.Helper() command := exec.Command( "python3", "-c", strings.Join([]string{ "import sqlite3, sys", "conn = sqlite3.connect(sys.argv[1])", "cur = conn.cursor()", "cur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')", "cur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')", "cur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")", "cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")", "cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")", "conn.commit()", "conn.close()", }, "; "), path, ) if output, err := command.CombinedOutput(); err != nil { t.Fatalf("create rosbag db3 %s: %v: %s", path, err, string(output)) } } func TestChooseImagePrefersRuntimeOperatorRunnerForBuiltins(t *testing.T) { node := domain.WorkflowNode{ Data: domain.WorkflowNodeData{ Image: "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:old-tag", }, } image := chooseImage(node, "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag") if image != "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag" { t.Fatalf("expected runtime image override for built-in operator-runner, got %q", image) } } func TestChooseImagePrefersDigestPinnedOperatorImageID(t *testing.T) { node := domain.WorkflowNode{ Data: domain.WorkflowNodeData{ Image: "ccr.example/dataflow-poc/droboticflow-operator-openvla-finetune:old", OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-openvla-finetune:fixed@sha256:" + strings.Repeat("a", 64), }, } image := chooseImage(node, "ccr.example/dataflow-poc/droboticflow-operator-runner:live") if image != node.Data.OperatorImageID { t.Fatalf("expected digest-pinned operator image id, got %q", image) } } func TestChooseImagePrefersUntaggedDigestPinnedOperatorImageID(t *testing.T) { node := domain.WorkflowNode{ Data: domain.WorkflowNodeData{ Image: "ccr.example/dataflow-poc/droboticflow-operator-act-policy-train:built-tag", OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-act-policy-train@sha256:" + strings.Repeat("b", 64), }, } image := chooseImage(node, "ccr.example/dataflow-poc/droboticflow-operator-runner:live") if image != node.Data.OperatorImageID { t.Fatalf("expected untagged digest-pinned operator image id, got %q", image) } } func TestResolveBuiltinNodeRuntimeRefreshesStaleWorkflowImage(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() currentImage := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:builtin" currentImageID := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("c", 64) if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{ ID: "operator-builtin-bos-incremental-scan", Name: "BOS 增量扫描", Type: "builtin", Category: "IO", Description: "scan", Image: currentImage, ImageID: currentImageID, ContainerCommand: []string{"python", "/app/backend/operator-runner/main.py"}, RuntimeContract: map[string]any{"handler": "builtin_source"}, }); err != nil { t.Fatalf("upsert builtin operator: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) node := domain.WorkflowNode{Data: domain.WorkflowNodeData{ OperatorID: "operator-builtin-bos-incremental-scan", Image: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:deleted", OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("a", 64), Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: "bos://bucket/prefix"}}, }} resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node) if err != nil { t.Fatalf("resolve builtin runtime: %v", err) } if resolved.Data.Image != currentImage || resolved.Data.OperatorImageID != currentImageID { t.Fatalf("expected current builtin image, got image=%q imageId=%q", resolved.Data.Image, resolved.Data.OperatorImageID) } if strings.Join(resolved.Data.ContainerCommand, " ") != "python /app/backend/operator-runner/main.py" { t.Fatalf("expected current container command, got %#v", resolved.Data.ContainerCommand) } if len(resolved.Data.Hyperparams) != 1 || resolved.Data.Hyperparams[0].Value != "bos://bucket/prefix" { t.Fatalf("expected workflow parameters to be preserved, got %#v", resolved.Data.Hyperparams) } } func TestResolveBuiltinNodeRuntimePreservesExplicitEnvironmentPin(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() currentImage := "registry.example/poc/operator:new" currentImageID := "registry.example/poc/operator@sha256:" + strings.Repeat("a", 64) if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{ID: "operator-builtin-package", Name: "Package", Type: "builtin", Image: currentImage, ImageID: currentImageID, ContainerCommand: []string{"python", "/app/backend/operator-runner/main.py"}}); err != nil { t.Fatal(err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) pinnedImage := "registry.example/fanyu/operator:release" pinnedImageID := "registry.example/fanyu/operator@sha256:" + strings.Repeat("b", 64) node := domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorID: "operator-builtin-package", Image: pinnedImage, OperatorImageID: pinnedImageID, Hyperparams: []domain.HyperParam{{Key: "PIN_OPERATOR_IMAGE", Value: "true"}}}} resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node) if err != nil { t.Fatal(err) } if resolved.Data.Image != pinnedImage || resolved.Data.OperatorImageID != pinnedImageID { t.Fatalf("environment pin was overwritten: %#v", resolved.Data) } if strings.Join(resolved.Data.ContainerCommand, " ") != "python /app/backend/operator-runner/main.py" { t.Fatalf("non-image runtime metadata was not refreshed: %#v", resolved.Data.ContainerCommand) } } func TestResolveBuiltinNodeRuntimeLeavesCustomOperatorSnapshotUntouched(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{ ID: "operator-custom", Name: "Custom", Type: "custom", Category: "IO", Description: "custom", Image: "registry.example/custom:new", ImageID: "registry.example/custom@sha256:" + strings.Repeat("d", 64), }); err != nil { t.Fatalf("upsert custom operator: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) node := domain.WorkflowNode{Data: domain.WorkflowNodeData{ OperatorID: "operator-custom", Image: "registry.example/custom:pinned", OperatorImageID: "registry.example/custom@sha256:" + strings.Repeat("e", 64), }} resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node) if err != nil { t.Fatalf("resolve custom runtime: %v", err) } if resolved.Data.Image != node.Data.Image || resolved.Data.OperatorImageID != node.Data.OperatorImageID { t.Fatalf("custom operator snapshot changed: %#v", resolved.Data) } } func createTestVideoMP4(t *testing.T, path string, pattern string, fps int, frames int, width int, height int) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) } command := exec.Command( "python3", "-c", strings.Join([]string{ "import cv2, numpy as np, sys", "path, pattern, fps, frames, width, height = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]), int(sys.argv[6])", "fourcc = cv2.VideoWriter_fourcc(*'mp4v')", "writer = cv2.VideoWriter(path, fourcc, fps, (width, height))", "assert writer.isOpened(), 'video writer did not open'", "for index in range(frames):", " frame = np.zeros((height, width, 3), dtype=np.uint8)", " if pattern == 'sharp':", " x = int((index * 3) % max(1, width - 18))", " cv2.rectangle(frame, (x, 8), (x + 16, 24), (255, 255, 255), -1)", " cv2.putText(frame, str(index % 10), (6, height - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1, cv2.LINE_AA)", " elif pattern == 'blur':", " cv2.rectangle(frame, (16, 16), (width - 16, height - 16), (220, 220, 220), -1)", " frame = cv2.GaussianBlur(frame, (23, 23), 0)", " else:", " x = int((index * 5) % max(1, width - 20))", " frame[:] = (40, 60, 90)", " cv2.circle(frame, (x + 10, height // 2), 10, (80, 140, 190), -1)", " writer.write(frame)", "writer.release()", "capture = cv2.VideoCapture(path)", "assert capture.isOpened(), 'video capture did not open'", "capture.release()", "print(path)", }, "\n"), path, pattern, fmt.Sprintf("%d", fps), fmt.Sprintf("%d", frames), fmt.Sprintf("%d", width), fmt.Sprintf("%d", height), ) if output, err := command.CombinedOutput(); err != nil { t.Fatalf("create test mp4 %s: %v: %s", path, err, string(output)) } } func createXPlatformDataset(t *testing.T) string { t.Helper() root := filepath.Join(t.TempDir(), "algo", "X-platform-test") fps := 10 width := 96 height := 64 type videoSpec struct { Name string Pattern string Frames int Duration float64 } type collectionSpec struct { Name string Type string Place string Date string DeviceID string HasIntrinsics bool VideoDirectory string Videos []videoSpec } collections := []collectionSpec{ { Name: "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", Type: "家庭", Place: "蓟门里小区", Date: "20260319", DeviceID: "C00000", HasIntrinsics: true, VideoDirectory: "DJI_001", Videos: []videoSpec{ {Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 40, Duration: 4.0}, {Name: "DJI_0002_D.mp4", Pattern: "blur", Frames: 20, Duration: 2.0}, }, }, { Name: "BJ_002_0002_OsmoNano_2026-03-20_10-30-00", Type: "办公", Place: "中关村", Date: "20260320", DeviceID: "C00001", HasIntrinsics: false, VideoDirectory: "DJI_001", Videos: []videoSpec{ {Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 20, Duration: 2.0}, }, }, } for _, collection := range collections { collectionRoot := filepath.Join(root, collection.Name) videoRoot := filepath.Join(collectionRoot, collection.VideoDirectory) videoMetaRows := make([]map[string]any, 0, len(collection.Videos)) for index, video := range collection.Videos { videoPath := filepath.Join(videoRoot, video.Name) createTestVideoMP4(t, videoPath, video.Pattern, fps, video.Frames, width, height) videoMetaRows = append(videoMetaRows, map[string]any{ "filename": video.Name, "codec": "mp4v", "resolution": fmt.Sprintf("%dx%d", width, height), "frameRate": float64(fps), "bitrate": 2000000 + index*100000, "duration": video.Duration, "startTime": 1773904218 + index*60, }) } metaJSON, err := json.Marshal(map[string]any{ "type": collection.Type, "place": collection.Place, "mission": "无", "project_id": collection.Name, "date": collection.Date, "device_id": collection.DeviceID, }) if err != nil { t.Fatalf("marshal meta json: %v", err) } writeFile(t, filepath.Join(collectionRoot, "meta.json"), string(metaJSON)) videoMetaJSON, err := json.Marshal(videoMetaRows) if err != nil { t.Fatalf("marshal video_meta json: %v", err) } writeFile(t, filepath.Join(collectionRoot, "video_meta.json"), string(videoMetaJSON)) writeFile(t, filepath.Join(videoRoot, "DJI_meta.json"), `{"ok":true}`) if collection.HasIntrinsics { writeFile(t, filepath.Join(collectionRoot, "intrinsics.json"), `{"fx": 1000, "fy": 1000}`) } } return root } func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord { t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { run, err := dataStore.GetRun(ctx, runID) if err != nil { t.Fatalf("get run %s: %v", runID, err) } if run != nil && run.Status == expected { return run } time.Sleep(100 * time.Millisecond) } run, err := dataStore.GetRun(ctx, runID) if err != nil { t.Fatalf("get run %s: %v", runID, err) } t.Fatalf("timed out waiting for run %s to become %s, got %#v", runID, expected, run) return nil } func waitForRunningNodeCount(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected int, timeout time.Duration) *domain.RunRecord { t.Helper() deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { run, err := dataStore.GetRun(ctx, runID) if err != nil { t.Fatalf("get run %s: %v", runID, err) } if run == nil { time.Sleep(100 * time.Millisecond) continue } running := 0 for _, execution := range run.NodeExecutions { if execution.Status == domain.NodeExecutionStatusRunning { running++ } } if running == expected { return run } time.Sleep(100 * time.Millisecond) } run, err := dataStore.GetRun(ctx, runID) if err != nil { t.Fatalf("get run %s: %v", runID, err) } t.Fatalf("timed out waiting for run %s to have %d running nodes, got %#v", runID, expected, run) return nil } func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Worker 项目", Description: "worker"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "批处理工作流", Description: "complete path", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "OSS 数据源", Category: "IO", Description: "source", Inputs: []domain.PortDefinition{}, Outputs: []domain.PortDefinition{ {ID: "out-1", Name: "file", Type: domain.PortTypeFile}, }, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "worker-runtime-test"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "sink", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", Category: "IO", Description: "sink", Inputs: []domain.PortDefinition{ {ID: "in-1", Name: "json", Type: domain.PortTypeJSON}, }, Outputs: []domain.PortDefinition{}, }, Position: domain.Position{X: 100, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "e1", Source: "source", SourceHandle: "out-1", Target: "sink", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if completed == nil || completed.Status != domain.RunStatusCompleted { t.Fatalf("expected completed run, got %#v", completed) } for _, item := range completed.NodeExecutions { if item.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected completed node execution, got %s", item.Status) } } if len(completed.Artifacts) == 0 { t.Fatalf("expected artifacts") } if completed.Artifacts[0].Path == "" { t.Fatalf("expected real artifact path") } if len(completed.ResourceSeries) == 0 { t.Fatalf("expected resource samples from operator runner") } } func TestRuntimeStopsGracefullyWithoutDispatchingNewNodes(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "停止项目", Description: "graceful"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "停止工作流", Description: "graceful stop path", Type: domain.WorkflowTypeBatch, MaxConcurrentRuns: 1, NodeTimeoutSeconds: 30, RunTimeoutSeconds: 60, MaxRetryAttempts: 0, RetryBackoffSeconds: 0, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "source", Outputs: []domain.PortDefinition{ {ID: "payload", Name: "payload", Type: domain.PortTypeJSON}, }, Hyperparams: []domain.HyperParam{ {Key: "COMMAND", Value: `sleep 1; printf '{"value":1}' > payload.json`}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "sink", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", Category: "IO", Description: "sink", Inputs: []domain.PortDefinition{ {ID: "input", Name: "input", Type: domain.PortTypeJSON}, }, }, Position: domain.Position{X: 120, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "e1", Source: "source", SourceHandle: "payload", Target: "sink", TargetHandle: "input"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } done := make(chan error, 1) go func() { _, processErr := runtime.ProcessOnce(ctx) done <- processErr }() waitForRunningNodeCount(t, ctx, dataStore, run.ID, 1, 5*time.Second) stored, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get running run: %v", err) } stored.Status = domain.RunStatus("stopping") stored.StopMode = domain.RunStopModeGraceful stored.StopRequestedAt = nowISO() stored.UpdatedAt = nowISO() if err := dataStore.UpsertRun(ctx, *stored); err != nil { t.Fatalf("request graceful stop: %v", err) } select { case err := <-done: if err != nil { t.Fatalf("process run: %v", err) } case <-time.After(10 * time.Second): t.Fatalf("timed out waiting for graceful stop") } finalRun, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get final run: %v", err) } if finalRun == nil || finalRun.Status != domain.RunStatusCancelled { t.Fatalf("expected cancelled run after graceful stop, got %#v", finalRun) } if sink := mustNodeExecution(t, finalRun, "source"); sink.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected source node to complete before stop, got %s", sink.Status) } for _, item := range finalRun.NodeExecutions { if item.NodeID == "sink" { t.Fatalf("expected sink node not to start after graceful stop, got %#v", item) } } } func TestRuntimePassesContainerCommandAndArgsToLauncher(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "镜像项目", Description: "command args"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "自定义容器入口", Description: "verify command args", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ ID: "custom", Type: "operator", Data: domain.WorkflowNodeData{ Label: "ACT 策略训练", Description: "custom entrypoint", Image: "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime", ContainerCommand: []string{"/bin/bash", "-lc"}, ContainerArgs: []string{"python /workspace/operators/act.py"}, RuntimeContract: map[string]any{"handler": "builtin_act_policy_train"}, }, Position: domain.Position{X: 0, Y: 0}, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } var captured k8s.LaunchInput runtime := &Runtime{ store: dataStore, operatorRunnerImage: "operator-runner:latest", workspaceRoot: t.TempDir(), launcher: testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { captured = input if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" { if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil { return nil, err } } return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: "custom", Namespace: "droboticflow"}, Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, nil }, }, } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } if captured.Image != "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime" { t.Fatalf("image = %q", captured.Image) } if strings.Join(captured.Command, " ") != "/bin/bash -lc" { t.Fatalf("command = %#v", captured.Command) } if len(captured.Args) != 1 || captured.Args[0] != "python /workspace/operators/act.py" { t.Fatalf("args = %#v", captured.Args) } if captured.Env["DROBOTICFLOW_OPERATOR_HANDLER"] != "act_policy_train" { t.Fatalf("handler env = %q", captured.Env["DROBOTICFLOW_OPERATOR_HANDLER"]) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if completed == nil || completed.Status != domain.RunStatusCompleted { t.Fatalf("expected completed run, got %#v", completed) } } func TestRuntimePassesLightweightOperatorMountToLauncher(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "轻量项目", Description: "lightweight"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "轻量算子工作流", Description: "verify lightweight code mount", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ ID: "light", Type: "operator", Data: domain.WorkflowNodeData{ Label: "轻量算子", Description: "code from workspace", Image: "python:3.11-slim", ContainerCommand: []string{"/bin/sh", "-lc"}, ContainerArgs: []string{"python operator.py"}, RuntimeContract: map[string]any{ "lightweightOperator": map[string]any{ "enabled": true, "codeSubPath": "lightweight-operators/demo", "mountPath": "/app/operator", "mounts": []any{map[string]any{"name": "data", "pvcName": "dataset-pvc", "subPath": "robotwin", "mountPath": "/data", "readOnly": true}}, }, }, }, Position: domain.Position{X: 0, Y: 0}, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { t.Fatalf("create run: %v", err) } var captured k8s.LaunchInput runtime := &Runtime{ store: dataStore, operatorRunnerImage: "operator-runner:latest", workspaceRoot: t.TempDir(), launcher: testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { captured = input if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" { if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil { return nil, err } } return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: "light", Namespace: "droboticflow"}, Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, nil }, }, } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } if captured.CodeSubPath != "lightweight-operators/demo" || captured.CodeMountPath != "/app/operator" { t.Fatalf("unexpected lightweight mount: subPath=%q mountPath=%q", captured.CodeSubPath, captured.CodeMountPath) } if captured.Env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] != "/app/operator" { t.Fatalf("LIGHTWEIGHT_OPERATOR_CODE_DIR = %q", captured.Env["LIGHTWEIGHT_OPERATOR_CODE_DIR"]) } if len(captured.PVCMounts) != 1 || captured.PVCMounts[0].PVCName != "dataset-pvc" || captured.PVCMounts[0].MountPath != "/data" || !captured.PVCMounts[0].ReadOnly { t.Fatalf("unexpected additional PVC mounts: %#v", captured.PVCMounts) } } func TestRuntimeRejectsLightweightOperatorWithoutExplicitContainerArgs(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "轻量项目", Description: "lightweight"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "轻量算子工作流", Description: "missing explicit args", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ ID: "light", Type: "operator", Data: domain.WorkflowNodeData{ Label: "轻量算子", Description: "code from workspace", Image: "python:3.11-slim", ContainerCommand: []string{"python", "/app/operator/operator.py"}, RuntimeContract: map[string]any{ "lightweightOperator": map[string]any{ "enabled": true, "codeSubPath": "lightweight-operators/demo", "mountPath": "/app/operator", }, }, }, Position: domain.Position{X: 0, Y: 0}, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { t.Fatalf("create run: %v", err) } launchCalled := false runtime := &Runtime{ store: dataStore, operatorRunnerImage: "operator-runner:latest", workspaceRoot: t.TempDir(), launcher: testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { launchCalled = true return nil, fmt.Errorf("launch should not be called") }, }, } processed := mustProcessOnceRun(t, runtime, ctx) if launchCalled { t.Fatalf("expected lightweight preflight to fail before launch") } if processed.Status != domain.RunStatusFailed || !strings.Contains(processed.Error, "containerArgs must be explicitly configured") { t.Fatalf("expected explicit args preflight failure, got status=%s error=%q", processed.Status, processed.Error) } } func TestRuntimePausesForAuditAndResumes(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Audit 项目", Description: "audit"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "审核工作流", Description: "audit path", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "review", Type: "operator", Data: domain.WorkflowNodeData{ Label: "人工审核 (Human Review)", Category: "HUMAN", Description: "review", Inputs: []domain.PortDefinition{ {ID: "in-1", Name: "json", Type: domain.PortTypeJSON}, }, Outputs: []domain.PortDefinition{ {ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", Category: "IO", Description: "sink", Inputs: []domain.PortDefinition{ {ID: "in-1", Name: "json", Type: domain.PortTypeJSON}, }, Outputs: []domain.PortDefinition{}, }, Position: domain.Position{X: 100, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "e1", Source: "review", SourceHandle: "approved", Target: "upload", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } paused, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get paused run: %v", err) } if paused == nil || paused.Status != domain.RunStatusWaitingForAudit { t.Fatalf("expected waiting_for_audit run, got %#v", paused) } tasks, err := dataStore.ListAuditTasks(ctx, "all") if err != nil { t.Fatalf("list audit tasks: %v", err) } var task *domain.AuditTaskRecord for _, item := range tasks { if item.RunID == run.ID && item.Status == domain.AuditStatusPending { copied := item task = &copied break } } if task == nil { t.Fatalf("expected pending audit task") } if task.OrganizationID != run.OrganizationID { t.Fatalf("audit task organization = %q, want run organization %q", task.OrganizationID, run.OrganizationID) } batches, err := dataStore.ListHumanBatches(ctx, run.OrganizationID, domain.HumanBatchQuery{RunID: run.ID, PluginID: "human-loop-core"}) if err != nil || len(batches) != 1 { t.Fatalf("generic human batches=%#v err=%v", batches, err) } humanItems, err := dataStore.ListHumanItems(ctx, run.OrganizationID, domain.HumanItemQuery{BatchID: batches[0].ID}) if err != nil || len(humanItems) != 1 || humanItems[0].LegacyAuditTaskID != task.ID { t.Fatalf("generic human items=%#v err=%v", humanItems, err) } task.Status = domain.AuditStatusApproved task.UpdatedAt = nowISO() if err := dataStore.UpsertAuditTask(ctx, *task); err != nil { t.Fatalf("approve audit task: %v", err) } paused.Status = domain.RunStatusQueued paused.UpdatedAt = nowISO() if err := dataStore.UpsertRun(ctx, *paused); err != nil { t.Fatalf("queue paused run: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("resume process once: %v", err) } resumed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get resumed run: %v", err) } if resumed == nil || resumed.Status != domain.RunStatusCompleted { t.Fatalf("expected completed resumed run, got %#v", resumed) } } func TestRuntimeRoutesArtifactsAcrossSequentialHumanReviews(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "双重审核项目", Description: "sequential audit"}) if err != nil { t.Fatalf("create project: %v", err) } reviewNode := func(id string, label string) domain.WorkflowNode { return domain.WorkflowNode{ ID: id, Type: "operator", Data: domain.WorkflowNodeData{ Label: label, Category: "HUMAN", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "candidate", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{ {ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON}, {ID: "rejected", Name: "审核拒绝", Type: domain.PortTypeJSON}, }, }, } } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "双重人工审核工作流", Description: "preserve artifacts through both gates", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: `{"items":[{"name":"clip.mp4"}],"count":1}`}}, }, }, reviewNode("quality-review", "切分与语义质检"), reviewNode("acceptance-review", "最终验收"), { ID: "quality-rejected-upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "质检拒绝上传(不应执行)", Category: "IO", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "rejected", Type: domain.PortTypeJSON}}, }, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-quality", Source: "source", SourceHandle: "out-1", Target: "quality-review", TargetHandle: "in-1"}, {ID: "quality-acceptance", Source: "quality-review", SourceHandle: "approved", Target: "acceptance-review", TargetHandle: "in-1"}, {ID: "quality-rejected", Source: "quality-review", SourceHandle: "rejected", Target: "quality-rejected-upload", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process to first audit: %v", err) } paused, err := dataStore.GetRun(ctx, run.ID) if err != nil || paused == nil { t.Fatalf("get first paused run: run=%#v err=%v", paused, err) } firstTasks, err := dataStore.ListAuditTasks(ctx, string(domain.AuditStatusPending)) if err != nil { t.Fatalf("list first audit tasks: %v", err) } if len(firstTasks) != 1 || firstTasks[0].NodeID != "quality-review" { t.Fatalf("expected quality review task, got %#v", firstTasks) } firstTasks[0].Status = domain.AuditStatusApproved firstTasks[0].UpdatedAt = nowISO() if err := dataStore.UpsertAuditTask(ctx, firstTasks[0]); err != nil { t.Fatalf("approve quality review: %v", err) } paused.Status = domain.RunStatusQueued paused.UpdatedAt = nowISO() if err := dataStore.UpsertRun(ctx, *paused); err != nil { t.Fatalf("queue after quality review: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process to final audit: %v", err) } finalRun, err := dataStore.GetRun(ctx, run.ID) if err != nil || finalRun == nil { t.Fatalf("get final audit run: run=%#v err=%v", finalRun, err) } qualityExecution := mustNodeExecution(t, finalRun, "quality-review") if len(qualityExecution.Artifacts) != 1 || qualityExecution.Artifacts[0].PortID != "approved" { t.Fatalf("expected approved routed artifact, got %#v", qualityExecution.Artifacts) } acceptanceExecution := mustNodeExecution(t, finalRun, "acceptance-review") if len(acceptanceExecution.InputArtifacts) != 1 { t.Fatalf("expected final review to receive one artifact, got %#v", acceptanceExecution.InputArtifacts) } rejectedExecution := mustNodeExecution(t, finalRun, "quality-rejected-upload") if rejectedExecution.Status != domain.NodeExecutionStatusCompleted || len(rejectedExecution.Artifacts) != 0 || len(rejectedExecution.InputArtifacts) != 0 { t.Fatalf("expected unselected rejection branch to be skipped without artifacts, got %#v", rejectedExecution) } if len(rejectedExecution.Logs) != 1 || !strings.Contains(rejectedExecution.Logs[0].Message, "Skipped by audit decision: approved") { t.Fatalf("expected explicit skipped-branch log, got %#v", rejectedExecution.Logs) } finalTasks, err := dataStore.ListAuditTasks(ctx, string(domain.AuditStatusPending)) if err != nil { t.Fatalf("list final audit tasks: %v", err) } if len(finalTasks) != 1 || finalTasks[0].NodeID != "acceptance-review" { t.Fatalf("expected final acceptance task, got %#v", finalTasks) } if len(finalTasks[0].Data.DetectedObjects) != 1 || finalTasks[0].Data.DetectedObjects[0] != "1 upstream artifact(s)" { t.Fatalf("expected final audit artifact summary, got %#v", finalTasks[0].Data.DetectedObjects) } } func TestRuntimeAnnotationGateCreatesTasksAndReturnsAcceptedRevisionArtifact(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: domain.DefaultOrganizationID, Name: "Native Annotation"}) if err != nil { t.Fatalf("create project: %v", err) } template := domain.AnnotationTemplateRecord{ // Organization-level published templates are reusable by tenant projects. ID: "template-1", OrganizationID: project.OrganizationID, ProjectID: "", Name: "动作模板", Version: 1, Status: domain.AnnotationTemplatePublished, Labels: []domain.SemanticLabelDefinition{{ID: "steady", Name: "平稳活动"}, {ID: "fast", Name: "快速移动"}}, HumanStages: domain.DefaultFanyuV58HumanStages(), } supplier := domain.SupplierRecord{ID: "supplier-a", OrganizationID: project.OrganizationID, Name: "供应商 A", Status: "active"} if err := dataStore.UpsertAnnotationTemplate(ctx, template); err != nil { t.Fatalf("upsert template: %v", err) } if err := dataStore.UpsertSupplier(ctx, supplier); err != nil { t.Fatalf("upsert supplier: %v", err) } for index, role := range []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator, domain.AnnotationRoleQualityReviewer} { if err := dataStore.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ ID: fmt.Sprintf("member-%d", index), OrganizationID: project.OrganizationID, ProjectID: project.ID, UserID: fmt.Sprintf("user-%d", index), SupplierID: supplier.ID, Roles: []domain.AnnotationAssignmentRole{role}, }); err != nil { t.Fatalf("upsert project member: %v", err) } } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ OrganizationID: project.OrganizationID, ProjectID: project.ID, Name: "Native Annotation", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ {ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "clips", Type: domain.PortTypeJSON}}}}, {ID: "annotation-gate", Type: "operator", Data: domain.WorkflowNodeData{ Label: "逐段语义标注", Category: "HUMAN", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "clips", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "approved", Name: "approved", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "REVIEW_MODE", Value: "annotation"}, {Key: "ANNOTATION_TEMPLATE_ID", Value: template.ID}, {Key: "ANNOTATION_SUPPLIER_ID", Value: supplier.ID}, }, }}, }, Edges: []domain.WorkflowEdge{{ID: "source-gate", Source: "source", SourceHandle: "out-1", Target: "annotation-gate", TargetHandle: "in-1"}}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } root := t.TempDir() videoPath := filepath.Join(root, "clip001.mp4") if err := os.WriteFile(videoPath, []byte("real-clip-placeholder"), 0o644); err != nil { t.Fatalf("write video fixture: %v", err) } clipsPath := filepath.Join(root, "clips-manifest.json") clipsPayload := map[string]any{ "clipCount": 1, "items": []any{map[string]any{ "clipId": "clip001", "clipName": "clip001.mp4", "clipPath": videoPath, "startSecond": 0.0, "endSecond": 10.0, "duration": 10.0, }}, } clipsBytes, _ := json.Marshal(clipsPayload) if err := os.WriteFile(clipsPath, clipsBytes, 0o644); err != nil { t.Fatalf("write clips manifest: %v", err) } prelabelPath := filepath.Join(root, "clip001_annotation.json") prelabelPayload := map[string]any{ "clipId": "clip001", "actions": []any{map[string]any{"label": "平稳活动", "start": 0.0, "end": 10.0, "confidence": 0.8}}, } prelabelBytes, _ := json.Marshal(prelabelPayload) if err := os.WriteFile(prelabelPath, prelabelBytes, 0o644); err != nil { t.Fatalf("write prelabel: %v", err) } run.Status = domain.RunStatusQueued run.CurrentNodeID = "" run.NodeExecutions = []domain.NodeExecutionRecord{{ NodeID: "source", NodeLabel: "source", Attempt: 1, Status: domain.NodeExecutionStatusCompleted, StartedAt: nowISO(), EndedAt: nowISO(), Logs: []domain.LogEntryRecord{}, ResourceSeries: []domain.ResourceSample{}, Artifacts: []domain.ExecutionArtifact{ {ID: "artifact-video", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "clip001.mp4", Type: domain.PortTypeVideo, Path: videoPath, Data: map[string]any{"preview": map[string]any{"duration": 10.0}}}, {ID: "artifact-headpose-render", RunID: run.ID, NodeID: "source", PortID: "headpose", Name: "headpose-annotated.mp4", Type: domain.PortTypeVideo, Path: videoPath, Data: map[string]any{"preview": map[string]any{"duration": 10.0}}}, {ID: "artifact-clips", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "clips-manifest.json", Type: domain.PortTypeJSON, Path: clipsPath, Data: clipsPayload}, {ID: "artifact-prelabel", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "clip001_annotation.json", Type: domain.PortTypeJSON, Path: prelabelPath, Data: prelabelPayload}, }, }} if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert seeded run: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) { return nil, fmt.Errorf("annotation gate should not launch a Kubernetes job") }}) runtime.workspaceRoot = root if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process to annotation gate: %v", err) } waiting, err := dataStore.GetRun(ctx, run.ID) if err != nil || waiting == nil || waiting.Status != domain.RunStatusWaitingForHuman { t.Fatalf("annotation gate did not pause run: run=%#v err=%v", waiting, err) } tasks, err := dataStore.ListAnnotationTasks(ctx, project.OrganizationID, project.ID) if err != nil || len(tasks) != 1 { t.Fatalf("annotation tasks = %#v err=%v", tasks, err) } if tasks[0].RunID != run.ID || tasks[0].NodeID != "annotation-gate" || tasks[0].ArtifactID != "artifact-video" || tasks[0].DataTaskID != run.TaskID { t.Fatalf("annotation task linkage = %#v", tasks[0]) } batches, err := dataStore.ListHumanBatches(ctx, project.OrganizationID, domain.HumanBatchQuery{RunID: run.ID}) if err != nil || len(batches) != 1 || batches[0].ReleasePolicy != domain.HumanReleaseAcceptedPartition { t.Fatalf("human batches = %#v err=%v", batches, err) } humanItems, err := dataStore.ListHumanItems(ctx, project.OrganizationID, domain.HumanItemQuery{BatchID: batches[0].ID}) if err != nil || len(humanItems) != 1 || humanItems[0].LegacyTaskID != tasks[0].ID || humanItems[0].CurrentStageID != domain.FanyuV58StageAnnotationCombined { t.Fatalf("human items = %#v err=%v", humanItems, err) } revisions, err := dataStore.ListAnnotationRevisions(ctx, project.OrganizationID, tasks[0].ID) if err != nil || len(revisions) != 1 || revisions[0].Revision != 0 || len(revisions[0].Segments) != 1 { t.Fatalf("prelabel revision = %#v err=%v", revisions, err) } acceptedSegments := []domain.AnnotationSegment{{ID: "segment-1", LabelID: "fast", Label: "快速移动", Start: 0, End: 10, Confidence: 1, Source: "manual"}} if err := dataStore.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ ID: "revision-accepted", OrganizationID: project.OrganizationID, TaskID: tasks[0].ID, Revision: 1, TemplateVersion: 1, Segments: acceptedSegments, AuthorUserID: "annotator", CountsAsOutput: true, CreatedAt: nowISO(), }); err != nil { t.Fatalf("upsert accepted revision: %v", err) } tasks[0].CurrentRevision = 1 tasks[0].Status = domain.AnnotationTaskStatusAccepted if err := dataStore.UpsertAnnotationTask(ctx, tasks[0]); err != nil { t.Fatalf("accept annotation task: %v", err) } waiting.Status = domain.RunStatusQueued if err := dataStore.UpsertRun(ctx, *waiting); err != nil { t.Fatalf("queue accepted annotation run: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("resume accepted annotation run: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil || completed == nil || completed.Status != domain.RunStatusCompleted { t.Fatalf("completed annotation run = %#v err=%v", completed, err) } gateExecution := mustNodeExecution(t, completed, "annotation-gate") manifest := mustArtifact(t, gateExecution, "annotations-manifest.json") if manifest.Metadata == nil || manifest.Metadata.SHA256 == "" || manifest.PortID != "approved" { t.Fatalf("accepted annotation artifact = %#v", manifest) } manifestItems := anySlice(manifest.Data["items"]) if len(manifestItems) != 1 { t.Fatalf("accepted manifest items = %#v", manifest.Data["items"]) } } func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Route 项目", Description: "route"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "条件分流工作流", Description: "route by handle", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "source", Outputs: []domain.PortDefinition{ {ID: "out-manifest", Name: "manifest", Type: domain.PortTypeJSON}, }, Hyperparams: []domain.HyperParam{ {Key: "COMMAND", Value: `printf '{"items":[{"kind":"keep","value":1},{"kind":"drop","value":2}]}' > routed.json`}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "route", Type: "operator", Data: domain.WorkflowNodeData{ Label: "条件分流", Category: "LOGIC", Description: "route", Inputs: []domain.PortDefinition{ {ID: "in-1", Name: "input", Type: domain.PortTypeJSON}, }, Outputs: []domain.PortDefinition{ {ID: "matched", Name: "matched", Type: domain.PortTypeJSON}, {ID: "unmatched", Name: "unmatched", Type: domain.PortTypeJSON}, }, Hyperparams: []domain.HyperParam{ {Key: "FIELD_PATH", Value: "kind"}, {Key: "OPERATOR", Value: "eq"}, {Key: "EXPECTED_JSON", Value: `"keep"`}, }, }, Position: domain.Position{X: 200, Y: 0}, }, { ID: "keep-upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", Category: "IO", Description: "keep sink", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, }, Position: domain.Position{X: 420, Y: -80}, }, { ID: "drop-upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", Category: "IO", Description: "drop sink", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, }, Position: domain.Position{X: 420, Y: 80}, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-route", Source: "source", SourceHandle: "out-manifest", Target: "route", TargetHandle: "in-1"}, {ID: "route-keep", Source: "route", SourceHandle: "matched", Target: "keep-upload", TargetHandle: "in-1"}, {ID: "route-drop", Source: "route", SourceHandle: "unmatched", Target: "drop-upload", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if completed == nil || completed.Status != domain.RunStatusCompleted { t.Fatalf("expected completed run, got %#v", completed) } routeExecution := mustNodeExecution(t, completed, "route") matched := mustArtifact(t, routeExecution, "matched.json") unmatched := mustArtifact(t, routeExecution, "unmatched.json") if matched.PortID != "matched" || unmatched.PortID != "unmatched" { t.Fatalf("expected output port IDs on split artifacts, got matched=%q unmatched=%q", matched.PortID, unmatched.PortID) } keepUpload := mustNodeExecution(t, completed, "keep-upload") dropUpload := mustNodeExecution(t, completed, "drop-upload") keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json") dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json") keepSummaryPayload := mustArtifactPayload(t, keepSummary) dropSummaryPayload := mustArtifactPayload(t, dropSummary) keepItems, _ := keepSummaryPayload["items"].([]any) dropItems, _ := dropSummaryPayload["items"].([]any) if len(keepItems) != 1 || len(dropItems) != 1 { t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems)) } keepPayload, _ := keepItems[0].(map[string]any) dropPayload, _ := dropItems[0].(map[string]any) keepRows, _ := keepPayload["items"].([]any) dropRows, _ := dropPayload["items"].([]any) if len(keepRows) != 1 || len(dropRows) != 1 { t.Fatalf("expected one row in each routed payload, got keep=%d drop=%d", len(keepRows), len(dropRows)) } if keepRow, _ := keepRows[0].(map[string]any); keepRow != nil { if keepRow["kind"] != "keep" { t.Fatalf("unexpected keep row: %#v", keepRow) } } if dropRow, _ := dropRows[0].(map[string]any); dropRow != nil { if dropRow["kind"] != "drop" { t.Fatalf("unexpected drop row: %#v", dropRow) } } } func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) { t.Run("command previews text and csv outputs", func(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Preview 项目", Description: "preview"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "文件预览工作流", Description: "preview files", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "command", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "command", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{ {Key: "COMMAND", Value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } commandExecution := mustNodeExecution(t, completed, "command") report := mustArtifact(t, commandExecution, "report.csv") notes := mustArtifact(t, commandExecution, "notes.txt") reportPreview, _ := report.Data["preview"].(map[string]any) notesPreview, _ := notes.Data["preview"].(map[string]any) if reportPreview["kind"] != "table" { t.Fatalf("expected table preview for csv artifact, got %#v", reportPreview) } if notesPreview["kind"] != "text" { t.Fatalf("expected text preview for text artifact, got %#v", notesPreview) } }) t.Run("lerobot dataset summary", func(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() datasetRoot := t.TempDir() writeFile(t, filepath.Join(datasetRoot, "meta", "info.json"), `{"robot_type":"aloha","fps":30}`) writeFile(t, filepath.Join(datasetRoot, "meta", "stats.json"), `{"episodes":2}`) writeFile(t, filepath.Join(datasetRoot, "meta", "tasks.jsonl"), "{\"task\":\"pick\"}\n{\"task\":\"place\"}\n") writeFile(t, filepath.Join(datasetRoot, "data", "episode_000.parquet"), "PAR1") writeFile(t, filepath.Join(datasetRoot, "videos", "episode_000.mp4"), "video") project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "LeRobot 项目", Description: "lerobot"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "LeRobot 概览", Description: "lerobot summary", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: datasetRoot}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "summary", Type: "operator", Data: domain.WorkflowNodeData{ Label: "LeRobot 数据集概览", Category: "ROBOTICS", Description: "lerobot", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}}, }, Position: domain.Position{X: 240, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json") summaryPayload := mustArtifactPayload(t, summary) preview, _ := summaryPayload["preview"].(map[string]any) if preview["format"] != "lerobot" { t.Fatalf("expected lerobot preview, got %#v", preview) } if summaryPayload["datasetType"] != "lerobot" { t.Fatalf("expected datasetType lerobot, got %#v", summaryPayload) } }) t.Run("rosbag sqlite summary", func(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() datasetRoot := t.TempDir() dbPath := filepath.Join(datasetRoot, "bag_0.db3") createRosbagDB3(t, dbPath) project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "ROS 项目", Description: "rosbag"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "ROSBag 概览", Description: "rosbag summary", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: dbPath}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "summary", Type: "operator", Data: domain.WorkflowNodeData{ Label: "ROSBag 数据概览", Category: "ROBOTICS", Description: "rosbag", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}}, }, Position: domain.Position{X: 240, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json") summaryPayload := mustArtifactPayload(t, summary) preview, _ := summaryPayload["preview"].(map[string]any) if preview["format"] != "rosbag2" { t.Fatalf("expected rosbag2 preview, got %#v", preview) } topics, _ := preview["topics"].([]any) if len(topics) != 1 { t.Fatalf("expected one topic summary, got %#v", topics) } }) } func TestRuntimeProcessesXPlatformPipeline(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() datasetRoot := createXPlatformDataset(t) project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "XPlatform 项目", Description: "x-platform"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "XPlatform 处理流水线", Description: "scan -> validate -> quality -> filter -> slice -> annotate -> package -> sample", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: datasetRoot}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "index", Type: "operator", Data: domain.WorkflowNodeData{ Label: "X 平台数据集索引", Category: "DATASET", Description: "index dataset", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "dataset", Type: domain.PortTypeJSON}}, }, Position: domain.Position{X: 220, Y: 0}, }, { ID: "validate", Type: "operator", Data: domain.WorkflowNodeData{ Label: "采集目录校验", Category: "DATASET", Description: "validate dataset", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "validated", Type: domain.PortTypeJSON}}, }, Position: domain.Position{X: 440, Y: 0}, }, { ID: "quality", Type: "operator", Data: domain.WorkflowNodeData{ Label: "视频质检", Category: "VIDEO", Description: "quality", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "quality", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "MIN_CLARITY", Value: "0.8"}, {Key: "MAX_SHAKE", Value: "0.1"}, {Key: "MAX_HAND_OCCLUSION", Value: "0.1"}, }, }, Position: domain.Position{X: 660, Y: 0}, }, { ID: "filter", Type: "operator", Data: domain.WorkflowNodeData{ Label: "数据过滤", Category: "VIDEO", Description: "filter dataset", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "filtered", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATE_START", Value: "2026-02-01"}, {Key: "DATE_END", Value: "2026-04-05"}, {Key: "PLACE", Value: "蓟门里小区"}, {Key: "TYPE", Value: "家庭"}, {Key: "MIN_CLARITY", Value: "0.8"}, {Key: "MAX_SHAKE", Value: "0.1"}, {Key: "MAX_HAND_OCCLUSION", Value: "0.1"}, }, }, Position: domain.Position{X: 880, Y: 0}, }, { ID: "slice", Type: "operator", Data: domain.WorkflowNodeData{ Label: "视频切片", Category: "VIDEO", Description: "slice video", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "clips", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "SEGMENT_DURATION_SECONDS", Value: "2"}}, }, Position: domain.Position{X: 1100, Y: 0}, }, { ID: "annotate", Type: "operator", Data: domain.WorkflowNodeData{ Label: "视频标注", Category: "VIDEO", Description: "annotate clips", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "FRAME_INTERVAL_SECONDS", Value: "0.5"}}, }, Position: domain.Position{X: 1320, Y: 0}, }, { ID: "package", Type: "operator", Data: domain.WorkflowNodeData{ Label: "成品打包", Category: "VIDEO", Description: "package results", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "packages", Type: domain.PortTypeJSON}}, }, Position: domain.Position{X: 1540, Y: 0}, }, { ID: "sample", Type: "operator", Data: domain.WorkflowNodeData{ Label: "抽检采样", Category: "QA", Description: "sample packages", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "RULES_JSON", Value: `[{"id":"clarity-check","field":"quality.clarity","operator":"lt","value":1,"samplePercent":100},{"id":"confidence-check","field":"annotation.minConfidence","operator":"lt","value":1,"samplePercent":100}]`}, }, }, Position: domain.Position{X: 1760, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-index", Source: "source", SourceHandle: "out-1", Target: "index", TargetHandle: "in-1"}, {ID: "index-validate", Source: "index", SourceHandle: "out-1", Target: "validate", TargetHandle: "in-1"}, {ID: "validate-quality", Source: "validate", SourceHandle: "out-1", Target: "quality", TargetHandle: "in-1"}, {ID: "quality-filter", Source: "quality", SourceHandle: "out-1", Target: "filter", TargetHandle: "in-1"}, {ID: "filter-slice", Source: "filter", SourceHandle: "out-1", Target: "slice", TargetHandle: "in-1"}, {ID: "slice-annotate", Source: "slice", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"}, {ID: "annotate-package", Source: "annotate", SourceHandle: "out-1", Target: "package", TargetHandle: "in-1"}, {ID: "package-sample", Source: "package", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if completed == nil || completed.Status != domain.RunStatusCompleted { t.Fatalf("expected completed run, got %#v", completed) } indexArtifact := mustArtifact(t, mustNodeExecution(t, completed, "index"), "dataset-index.json") indexPayload := mustArtifactPayload(t, indexArtifact) if indexPayload["collectionCount"] != float64(2) { t.Fatalf("expected collectionCount=2, got %#v", indexPayload) } if indexPayload["videoCount"] != float64(3) { t.Fatalf("expected videoCount=3, got %#v", indexPayload) } validateArtifact := mustArtifact(t, mustNodeExecution(t, completed, "validate"), "dataset-validation.json") validatePayload := mustArtifactPayload(t, validateArtifact) if validatePayload["validCollectionCount"] != float64(2) { t.Fatalf("expected validCollectionCount=2, got %#v", validatePayload) } qualityArtifact := mustArtifact(t, mustNodeExecution(t, completed, "quality"), "quality-report.json") qualityPayload := mustArtifactPayload(t, qualityArtifact) if qualityPayload["videoCount"] != float64(3) { t.Fatalf("expected quality videoCount=3, got %#v", qualityPayload) } if qualityPayload["passedCount"] != float64(2) { t.Fatalf("expected passedCount=2, got %#v", qualityPayload) } filterArtifact := mustArtifact(t, mustNodeExecution(t, completed, "filter"), "filtered-videos.json") filterPayload := mustArtifactPayload(t, filterArtifact) if filterPayload["count"] != float64(1) { t.Fatalf("expected filter count=1, got %#v", filterPayload) } sliceExecution := mustNodeExecution(t, completed, "slice") clipsArtifact := mustArtifact(t, sliceExecution, "clips-manifest.json") clipsPayload := mustArtifactPayload(t, clipsArtifact) if clipsPayload["clipCount"] != float64(2) { t.Fatalf("expected clipCount=2, got %#v", clipsPayload) } // 虚拟切片(VIRTUAL_CUT 默认开启)只记录时间戳区间,不再产出物理片段 MP4; // 真实裁剪由打包阶段按源视频区间统一物化。 if clipsPayload["virtualCut"] != true { t.Fatalf("expected clips manifest virtualCut=true, got %#v", clipsPayload["virtualCut"]) } clipVideos := 0 for _, artifact := range sliceExecution.Artifacts { if strings.HasSuffix(artifact.Name, ".mp4") { clipVideos++ } } if clipVideos != 0 { t.Fatalf("expected 0 physical clip videos under virtual cutting, got %d", clipVideos) } annotateExecution := mustNodeExecution(t, completed, "annotate") annotationsArtifact := mustArtifact(t, annotateExecution, "annotations-manifest.json") annotationsPayload := mustArtifactPayload(t, annotationsArtifact) if annotationsPayload["annotationCount"] != float64(2) { t.Fatalf("expected annotationCount=2, got %#v", annotationsPayload) } packageExecution := mustNodeExecution(t, completed, "package") packagesArtifact := mustArtifact(t, packageExecution, "packages-manifest.json") packagesPayload := mustArtifactPayload(t, packagesArtifact) if packagesPayload["packageCount"] != float64(2) { t.Fatalf("expected packageCount=2, got %#v", packagesPayload) } packageFiles := 0 for _, artifact := range packageExecution.Artifacts { if strings.Contains(artifact.Path, "/packages/") && (strings.HasSuffix(artifact.Name, ".json") || strings.HasSuffix(artifact.Name, ".mp4")) { packageFiles++ } } if packageFiles < 8 { t.Fatalf("expected packaged files to be emitted, got %d artifacts", packageFiles) } sampleArtifact := mustArtifact(t, mustNodeExecution(t, completed, "sample"), "audit-samples.json") samplePayload := mustArtifactPayload(t, sampleArtifact) if samplePayload["sampledCount"] != float64(2) { t.Fatalf("expected sampledCount=2, got %#v", samplePayload) } } func TestRuntimeCommitsIncrementalSourceStateOnlyOnSuccessfulRuns(t *testing.T) { ctx := context.Background() t.Run("commits state after successful run and next run only emits new files", func(t *testing.T) { dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "增量项目", Description: "incremental"}) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "增量源工作流", Description: "incremental source", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "INCREMENTAL"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, Edges: []domain.WorkflowEdge{}, }) if err != nil { t.Fatalf("create workflow: %v", err) } runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create first run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process first run: %v", err) } completedOne, err := dataStore.GetRun(ctx, runOne.ID) if err != nil { t.Fatalf("get first run: %v", err) } manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json") manifestOnePayload := mustArtifactPayload(t, manifestOne) if manifestOnePayload["newCount"] != float64(1) { t.Fatalf("expected newCount=1 on first run, got %#v", manifestOnePayload) } state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestOnePayload["sourceFingerprint"].(string))) if err != nil { t.Fatalf("get source state: %v", err) } if state == nil || len(state.SeenFingerprints) != 1 { t.Fatalf("expected committed source state after first run, got %#v", state) } writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`) runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) if err != nil { t.Fatalf("create second run: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process second run: %v", err) } completedTwo, err := dataStore.GetRun(ctx, runTwo.ID) if err != nil { t.Fatalf("get second run: %v", err) } manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json") manifestTwoPayload := mustArtifactPayload(t, manifestTwo) if manifestTwoPayload["listedCount"] != float64(2) { t.Fatalf("expected listedCount=2 on second run, got %#v", manifestTwoPayload) } if manifestTwoPayload["newCount"] != float64(1) { t.Fatalf("expected newCount=1 on second run, got %#v", manifestTwoPayload) } entries, ok := manifestTwoPayload["entries"].([]any) if !ok || len(entries) != 1 { t.Fatalf("expected one emitted entry on second run, got %#v", manifestTwoPayload["entries"]) } entry, ok := entries[0].(map[string]any) if !ok || !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/meta.json") { t.Fatalf("expected second run to emit only newly added file, got %#v", entries[0]) } state, err = dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string))) if err != nil { t.Fatalf("get source state after second run: %v", err) } if state == nil || len(state.SeenFingerprints) != 2 { t.Fatalf("expected 2 seen fingerprints after second run, got %#v", state) } if state.LastNewCount != 1 || state.LastListedCount != 2 { t.Fatalf("unexpected source state stats after second run: %#v", state) } }) t.Run("directory incremental mode emits entire newly discovered directories", func(t *testing.T) { dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "目录增量项目", Description: "incremental-directory"}) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-a.mp4"), "a") workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "目录增量源工作流", Description: "incremental directory source", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "INCREMENTAL"}, {Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"}, {Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "1"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, }, Edges: []domain.WorkflowEdge{}, }) if err != nil { t.Fatalf("create workflow: %v", err) } runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create first run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process first run: %v", err) } completedOne, err := dataStore.GetRun(ctx, runOne.ID) if err != nil { t.Fatalf("get first run: %v", err) } manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json") manifestOnePayload := mustArtifactPayload(t, manifestOne) if manifestOnePayload["newCount"] != float64(2) { t.Fatalf("expected first run to emit full batch-a directory, got %#v", manifestOnePayload["newCount"]) } if manifestOnePayload["incrementalUnit"] != "DIRECTORY" { t.Fatalf("expected directory incremental unit, got %#v", manifestOnePayload["incrementalUnit"]) } writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-b.mp4"), "b") writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`) writeFile(t, filepath.Join(sourceRoot, "batch-b", "DJI_001", "clip-c.mp4"), "c") runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) if err != nil { t.Fatalf("create second run: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process second run: %v", err) } completedTwo, err := dataStore.GetRun(ctx, runTwo.ID) if err != nil { t.Fatalf("get second run: %v", err) } manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json") manifestTwoPayload := mustArtifactPayload(t, manifestTwo) if manifestTwoPayload["listedCount"] != float64(5) { t.Fatalf("expected listedCount=5 on second run, got %#v", manifestTwoPayload["listedCount"]) } if manifestTwoPayload["newCount"] != float64(2) { t.Fatalf("expected newCount=2 on second run, got %#v", manifestTwoPayload["newCount"]) } entries, ok := manifestTwoPayload["entries"].([]any) if !ok || len(entries) != 2 { t.Fatalf("expected two emitted entries for new directory, got %#v", manifestTwoPayload["entries"]) } for _, rawEntry := range entries { entry, ok := rawEntry.(map[string]any) if !ok { t.Fatalf("unexpected entry payload: %#v", rawEntry) } if !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/") { t.Fatalf("expected second run to emit only newly discovered directory contents, got %#v", entry["path"]) } } state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string))) if err != nil { t.Fatalf("get source state after second run: %v", err) } if state == nil || len(state.SeenFingerprints) != 2 { t.Fatalf("expected two seen directory keys after second run, got %#v", state) } if state.IncrementalUnit != "DIRECTORY" || state.DirectoryDepth != 1 { t.Fatalf("expected committed directory incremental state, got %#v", state) } }) t.Run("does not commit state when downstream run fails", func(t *testing.T) { dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "失败增量项目", Description: "incremental-fail"}) if err != nil { t.Fatalf("create project: %v", err) } sourceRoot := t.TempDir() writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`) workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "增量失败工作流", Description: "incremental fail", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "LOCAL"}, {Key: "SOURCE_PATH", Value: sourceRoot}, {Key: "SCAN_MODE", Value: "INCREMENTAL"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "fail", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "fail", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "echo boom >&2; exit 1"}}, }, Position: domain.Position{X: 240, Y: 0}, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-fail", Source: "source", SourceHandle: "out-1", Target: "fail", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process failed run: %v", err) } failedRun, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get failed run: %v", err) } if failedRun == nil || failedRun.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %#v", failedRun) } manifest := mustArtifact(t, mustNodeExecution(t, failedRun, "source"), "source-manifest.json") manifestPayload := mustArtifactPayload(t, manifest) state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestPayload["sourceFingerprint"].(string))) if err != nil { t.Fatalf("get source state after failed run: %v", err) } if state != nil { t.Fatalf("expected no committed source state after failed run, got %#v", state) } }) } func TestInputStorageConnectionIDUsesDirectoryReferenceAndRejectsAmbiguity(t *testing.T) { payload := domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeFile, Ref: &domain.ArtifactReference{ConnectionID: "storage-source", Bucket: "bucket", Prefix: "capture", Kind: domain.ArtifactRefKindDirectory}} connectionID, err := inputStorageConnectionID([]domain.NodeInputBinding{{TargetPortID: "in-1", Payload: &payload}}) if err != nil || connectionID != "storage-source" { t.Fatalf("connectionID=%q err=%v", connectionID, err) } other := payload refCopy := *payload.Ref refCopy.ConnectionID = "storage-other" other.Ref = &refCopy if _, err := inputStorageConnectionID([]domain.NodeInputBinding{{Payload: &payload}, {Payload: &other}}); err == nil { t.Fatal("multiple input storage connections must fail closed") } } func TestPrepareExecutionNodeResolvesStorageConnectionSecrets(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ Name: "BOS 主连接", Provider: domain.StorageProviderBOS, Endpoint: "https://gz.bcebos.com", AccessKeyID: "ak-runtime-1234", SecretAccessKey: "sk-runtime-5678", }) if err != nil { t.Fatalf("create storage connection: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } workflow := &domain.WorkflowRecord{ ID: "workflow-scan", Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: "BOS", ConnectionID: connection.ID, Bucket: "datastorage", Prefix: "algo/X-platform-test", ScanMode: "INCREMENTAL", }, } node := domain.WorkflowNode{ ID: "node-bos-scan-runtime", Type: "operator", Data: domain.WorkflowNodeData{ Label: "BOS 增量扫描", Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "BOS"}, }, }, } prepared, err := runtime.prepareExecutionNode(ctx, workflow, node) if err != nil { t.Fatalf("prepare execution node: %v", err) } params := nodeHyperparamsMap(prepared) if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" { t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"]) } if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-5678" { t.Fatalf("expected resolved bos secrets, got %#v", params) } if params["SOURCE_PATH"] != "bos://datastorage/algo/X-platform-test" { t.Fatalf("expected source path from bucket/prefix, got %q", params["SOURCE_PATH"]) } } func TestPrepareExecutionNodeUsesRuntimeGlobalBOSEndpointForConnectionWithoutEndpoint(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ Name: "BOS 全局 endpoint 连接", Provider: domain.StorageProviderBOS, AccessKeyID: "ak-runtime-default", SecretAccessKey: "sk-runtime-default", }) if err != nil { t.Fatalf("create storage connection: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } runtime.globalBOSEndpoint = "https://gz.bcebos.com" workflow := &domain.WorkflowRecord{ ID: "workflow-default-bos", Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: "BOS", ConnectionID: connection.ID, Bucket: "datastorage", Prefix: "algo/default-endpoint", ScanMode: "INCREMENTAL", }, } node := domain.WorkflowNode{ ID: "node-default-bos", Type: "operator", Data: domain.WorkflowNodeData{ Label: "BOS 增量扫描", Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "BOS"}}, }, } prepared, err := runtime.prepareExecutionNode(ctx, workflow, node) if err != nil { t.Fatalf("prepare execution node: %v", err) } params := nodeHyperparamsMap(prepared) if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" { t.Fatalf("expected runtime global bos endpoint, got %q", params["BOS_ENDPOINT"]) } if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-default" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-default" { t.Fatalf("expected resolved bos secrets, got %#v", params) } } func TestPrepareExecutionNodeKeepsEmptyEndpointWithoutGlobalBOSFallback(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ Name: "BOS 无 endpoint 连接", Provider: domain.StorageProviderBOS, AccessKeyID: "ak-runtime-empty-endpoint", SecretAccessKey: "sk-runtime-empty-endpoint", }) if err != nil { t.Fatalf("create storage connection: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } workflow := &domain.WorkflowRecord{ ID: "workflow-empty-bos-endpoint", Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: "BOS", ConnectionID: connection.ID, Bucket: "datastorage", Prefix: "algo/empty-endpoint", ScanMode: "INCREMENTAL", }, } node := domain.WorkflowNode{ ID: "node-empty-bos-endpoint", Type: "operator", Data: domain.WorkflowNodeData{ Label: "BOS 增量扫描", Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "BOS"}}, }, } prepared, err := runtime.prepareExecutionNode(ctx, workflow, node) if err != nil { t.Fatalf("prepare execution node: %v", err) } params := nodeHyperparamsMap(prepared) if params["BOS_ENDPOINT"] != "" { t.Fatalf("expected empty endpoint without global fallback, got %q", params["BOS_ENDPOINT"]) } if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-empty-endpoint" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-empty-endpoint" { t.Fatalf("expected resolved bos secrets, got %#v", params) } } func TestPrepareExecutionNodeResolvesNodeLevelStorageConnectionSecrets(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ Name: "BOS 节点连接", Provider: domain.StorageProviderBOS, Endpoint: "https://su.bcebos.com", AccessKeyID: "ak-node-1234", SecretAccessKey: "sk-node-5678", }) if err != nil { t.Fatalf("create storage connection: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } node := domain.WorkflowNode{ ID: "node-oss-source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "OSS 数据源", ConnectionID: connection.ID, Bucket: "datastorage", Prefix: "factory/output", Hyperparams: []domain.HyperParam{ {Key: "SOURCE_PATH", Value: ""}, {Key: "DATA_SOURCE", Value: "BOS"}, }, }, } prepared, err := runtime.prepareExecutionNode(ctx, nil, node) if err != nil { t.Fatalf("prepare execution node: %v", err) } params := nodeHyperparamsMap(prepared) if params["DATA_SOURCE"] != "BOS" { t.Fatalf("expected BOS data source, got %q", params["DATA_SOURCE"]) } if params["BOS_ENDPOINT"] != "https://su.bcebos.com" { t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"]) } if params["BOS_ACCESS_KEY_ID"] != "ak-node-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-node-5678" { t.Fatalf("expected resolved node bos secrets, got %#v", params) } if params["SOURCE_PATH"] != "bos://datastorage/factory/output" { t.Fatalf("expected source path from node bucket/prefix, got %q", params["SOURCE_PATH"]) } } func TestPrepareExecutionNodeResolvesMongoConnectionSecrets(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{ Name: "业务 MongoDB", URI: "mongodb://business-mongo:27017", Database: "business_data", }) if err != nil { t.Fatalf("create mongo connection: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } node := domain.WorkflowNode{ ID: "node-mongo-write", Type: "operator", Data: domain.WorkflowNodeData{ Label: "MongoDB 写入", Hyperparams: []domain.HyperParam{ {Key: "MONGO_CONNECTION_ID", Value: connection.ID}, {Key: "MONGO_COLLECTION", Value: "qualityReports"}, }, }, } prepared, err := runtime.prepareExecutionNode(ctx, nil, node) if err != nil { t.Fatalf("prepare execution node: %v", err) } params := nodeHyperparamsMap(prepared) if params["MONGO_URI"] != "mongodb://business-mongo:27017" { t.Fatalf("expected resolved mongo uri, got %q", params["MONGO_URI"]) } if params["MONGO_DB"] != "business_data" { t.Fatalf("expected resolved mongo db, got %q", params["MONGO_DB"]) } if params["MONGO_TARGET"] != "business" { t.Fatalf("expected mongo target business, got %q", params["MONGO_TARGET"]) } } func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() settings, err := dataStore.GetSettings(ctx) if err != nil { t.Fatalf("get settings: %v", err) } settings.Worker.MaxConcurrentRuns = 2 if err := dataStore.UpsertSettings(ctx, *settings); err != nil { t.Fatalf("upsert settings: %v", err) } project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Dispatch 项目", Description: "dispatch"}) if err != nil { t.Fatalf("create project: %v", err) } createWorkflow := func(name string) *domain.WorkflowRecord { t.Helper() workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: name, Description: "dispatch test", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "command", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "command", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}}, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow %s: %v", name, err) } return workflow } workflowA := createWorkflow("并发工作流 A") workflowB := createWorkflow("并发工作流 B") runA, err := dataStore.CreateRun(ctx, workflowA.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run A: %v", err) } runB, err := dataStore.CreateRun(ctx, workflowB.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run B: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } runs, err := dataStore.ListRuns(ctx, "") if err != nil { t.Fatalf("list runs: %v", err) } runningCount := 0 for _, run := range runs { if run.Status == domain.RunStatusRunning { runningCount++ } } if runningCount != 2 { t.Fatalf("expected 2 running runs after dispatch, got %d", runningCount) } waitForRunStatus(t, ctx, dataStore, runA.ID, domain.RunStatusCompleted, 8*time.Second) waitForRunStatus(t, ctx, dataStore, runB.ID, domain.RunStatusCompleted, 8*time.Second) } func TestSelectQueuedRunsLetsNewCapturePassAuditWaits(t *testing.T) { settings := &domain.SettingsRecord{} settings.Worker.MaxConcurrentRuns = 2 workflows := []domain.WorkflowRecord{ {ID: "wf-human", MaxConcurrentRuns: 1}, {ID: "wf-data"}, {ID: "wf-video"}, } runs := []domain.RunRecord{ {ID: "audit-wait", WorkflowID: "wf-human", Status: domain.RunStatusWaitingForAudit}, {ID: "human-next", WorkflowID: "wf-human", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:01Z"}, {ID: "data-next", WorkflowID: "wf-data", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:02Z"}, {ID: "video-next", WorkflowID: "wf-video", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:03Z"}, } selected, queueDepth, activeTotal := selectQueuedRunsForDispatchWithStats(runs, workflows, settings) if activeTotal != 0 { t.Fatalf("waiting_for_audit should not consume global dispatch slots, activeTotal=%d", activeTotal) } if queueDepth != 3 { t.Fatalf("queueDepth = %d, want 3", queueDepth) } if len(selected) != 2 { t.Fatalf("selected = %#v, want two runs", selected) } if selected[0] != "human-next" || selected[1] != "data-next" { t.Fatalf("selected = %#v, want the next Capture plus the next workflow in FIFO order", selected) } } func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() settings, err := dataStore.GetSettings(ctx) if err != nil { t.Fatalf("get settings: %v", err) } settings.Worker.MaxConcurrentRuns = 3 if err := dataStore.UpsertSettings(ctx, *settings); err != nil { t.Fatalf("upsert settings: %v", err) } project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Override 项目", Description: "override"}) if err != nil { t.Fatalf("create project: %v", err) } limitedWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "受限工作流", Description: "limited", Type: domain.WorkflowTypeBatch, MaxConcurrentRuns: 1, Nodes: []domain.WorkflowNode{{ ID: "command", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "command", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}}, }, Position: domain.Position{X: 0, Y: 0}, }}, }) if err != nil { t.Fatalf("create limited workflow: %v", err) } freeWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "自由工作流", Description: "free", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ ID: "command", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "command", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}}, }, Position: domain.Position{X: 0, Y: 0}, }}, }) if err != nil { t.Fatalf("create free workflow: %v", err) } runLimitedA, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create limited run A: %v", err) } runLimitedB, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create limited run B: %v", err) } runFree, err := dataStore.CreateRun(ctx, freeWorkflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create free run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } limitedRunAState, err := dataStore.GetRun(ctx, runLimitedA.ID) if err != nil { t.Fatalf("get limited run A: %v", err) } limitedRunBState, err := dataStore.GetRun(ctx, runLimitedB.ID) if err != nil { t.Fatalf("get limited run B: %v", err) } freeRunState, err := dataStore.GetRun(ctx, runFree.ID) if err != nil { t.Fatalf("get free run: %v", err) } if limitedRunAState.Status != domain.RunStatusRunning { t.Fatalf("expected first limited run to be running, got %s", limitedRunAState.Status) } if limitedRunBState.Status != domain.RunStatusQueued { t.Fatalf("expected second limited run to remain queued, got %s", limitedRunBState.Status) } if freeRunState.Status != domain.RunStatusRunning { t.Fatalf("expected free workflow run to be running, got %s", freeRunState.Status) } waitForRunStatus(t, ctx, dataStore, runLimitedA.ID, domain.RunStatusCompleted, 8*time.Second) waitForRunStatus(t, ctx, dataStore, runFree.ID, domain.RunStatusCompleted, 8*time.Second) if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available second pass: %v", err) } waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second) } func TestRuntimeProcessAvailableSkipsStreamingWorkflowWithMissingProject(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "有效项目", Description: "valid"}) if err != nil { t.Fatalf("create project: %v", err) } validWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "正常工作流", Description: "valid workflow", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "command", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "command", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > result.json"}}, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create valid workflow: %v", err) } run, err := dataStore.CreateRun(ctx, validWorkflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create queued run: %v", err) } orphanWorkflow := domain.WorkflowRecord{ ID: "workflow-orphan-streaming", ProjectID: "project-missing", Name: "孤儿增量工作流", Description: "orphan streaming workflow", Type: domain.WorkflowTypeStreaming, Schedule: &domain.WorkflowSchedule{Interval: 30}, Nodes: []domain.WorkflowNode{}, Edges: []domain.WorkflowEdge{}, Status: domain.WorkflowStatusDraft, CreatedAt: nowISO(), UpdatedAt: nowISO(), } if err := dataStore.UpsertWorkflow(ctx, orphanWorkflow); err != nil { t.Fatalf("upsert orphan workflow: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available should skip orphan workflow instead of failing: %v", err) } waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) } func TestShouldEnqueueScheduledRunEveryHoursUsesAnchorTime(t *testing.T) { workflow := domain.WorkflowRecord{ ID: "workflow-hours", Type: domain.WorkflowTypeStreaming, Schedule: &domain.WorkflowSchedule{ Mode: domain.WorkflowScheduleModeEveryHours, IntervalHours: 4, AnchorAt: "2026-04-21T10:15:00Z", }, } if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 15, 0, 0, time.UTC)) { t.Fatalf("expected first hourly window to enqueue at anchor time") } workflow.LastRunAt = "2026-04-21T10:15:00Z" if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 14, 59, 0, time.UTC)) { t.Fatalf("expected hourly schedule to wait until full interval elapsed") } if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 15, 0, 0, time.UTC)) { t.Fatalf("expected hourly schedule to enqueue on the next anchored slot") } } func TestShouldEnqueueScheduledRunEveryDaysAtFixedTimeUsesAnchorDate(t *testing.T) { workflow := domain.WorkflowRecord{ ID: "workflow-days", Type: domain.WorkflowTypeStreaming, Schedule: &domain.WorkflowSchedule{ Mode: domain.WorkflowScheduleModeEveryDaysAtTime, DayInterval: 2, TimeOfDay: "18:30", Timezone: "Asia/Shanghai", AnchorAt: "2026-04-21T09:00:00Z", }, } if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 29, 59, 0, time.UTC)) { t.Fatalf("expected daily schedule to wait until configured local time") } if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 30, 0, 0, time.UTC)) { t.Fatalf("expected daily schedule to enqueue at configured local time") } workflow.LastRunAt = "2026-04-21T10:30:00Z" if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 29, 59, 0, time.UTC)) { t.Fatalf("expected next two-day slot to remain pending before local HH:mm") } if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 30, 0, 0, time.UTC)) { t.Fatalf("expected next two-day slot to enqueue at configured local HH:mm") } } func TestRuntimeProcessRunResumesQueuedPartialRunFromCompletedDependencies(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "恢复工作流", Description: "resume", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}}, }, }, { ID: "transform", Type: "operator", Data: domain.WorkflowNodeData{ Label: "JSON 映射重命名", Category: "DATA", Description: "transform", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "FROM_KEY", Value: "items"}, {Key: "TO_KEY", Value: "entries"}}, }, }, { ID: "upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "sink", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}}, }, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-transform", Source: "source", SourceHandle: "out-1", Target: "transform", TargetHandle: "in-1"}, {ID: "transform-upload", Source: "transform", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } run.Status = domain.RunStatusQueued run.CurrentNodeID = "" run.NodeExecutions = []domain.NodeExecutionRecord{ { NodeID: "source", NodeLabel: "真实数据源", Status: domain.NodeExecutionStatusCompleted, StartedAt: nowISO(), EndedAt: nowISO(), Logs: []domain.LogEntryRecord{}, Artifacts: []domain.ExecutionArtifact{{ID: "artifact-source", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "payload.json", Type: domain.PortTypeJSON, Path: filepath.Join(t.TempDir(), "payload.json"), Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1}}}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, }, } if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) if execution := mustNodeExecution(t, completed, "transform"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected transform to complete after resume, got %s", execution.Status) } if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected upload to complete after resume, got %s", execution.Status) } } func TestRuntimeReconcileActiveRunsRequeuesClaimedRunWithoutJobs(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "僵尸运行恢复", Description: "recover", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } run.Status = domain.RunStatusRunning run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano) run.UpdatedAt = run.StartedAt run.NodeExecutions = []domain.NodeExecutionRecord{} if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile active runs: %v", err) } recovered, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get recovered run: %v", err) } if recovered == nil || recovered.Status != domain.RunStatusQueued { t.Fatalf("expected run to be re-queued, got %#v", recovered) } if recovered.StartedAt != "" { t.Fatalf("expected startedAt cleared after requeue, got %q", recovered.StartedAt) } } func TestRuntimeReconcileActiveRunsClearsQueuedRunWithMissingActiveJob(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover queued active"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "队列运行节点恢复", Description: "recover queued active", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } startedAt := time.Now().UTC().Add(-10 * time.Minute).Format(time.RFC3339Nano) run.Status = domain.RunStatusQueued run.CurrentNodeID = "source" run.StartedAt = startedAt run.UpdatedAt = startedAt run.NodeExecutions = []domain.NodeExecutionRecord{{ NodeID: "source", NodeLabel: "真实数据源", Status: domain.NodeExecutionStatusRunning, StartedAt: startedAt, Logs: []domain.LogEntryRecord{{Timestamp: startedAt, Level: "INFO", Message: "Starting node execution", Operator: "真实数据源"}}, }} if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile active runs: %v", err) } recovered, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get recovered run: %v", err) } if recovered == nil || recovered.Status != domain.RunStatusQueued { t.Fatalf("expected run to remain queued, got %#v", recovered) } if recovered.CurrentNodeID != "" { t.Fatalf("expected current node cleared, got %q", recovered.CurrentNodeID) } if len(recovered.NodeExecutions) != 1 || recovered.NodeExecutions[0].Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected stale source execution retained as failed recovery evidence, got %#v", recovered.NodeExecutions) } if !strings.Contains(recovered.NodeExecutions[0].Error, "recovery attempt 1/3") { t.Fatalf("expected recovery attempt evidence, got %#v", recovered.NodeExecutions[0]) } // Recovery deliberately applies a short backoff in production. Make this // fixture dispatchable now so the test can verify the resumed execution. recovered.AvailableAt = nowISO() if err := dataStore.UpsertRun(ctx, *recovered); err != nil { t.Fatalf("make recovered run dispatchable: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once after recovery: %v", err) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) if execution := mustNodeExecution(t, completed, "source"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected source to complete after stale execution recovery, got %s", execution.Status) } } func TestRuntimeProcessRunResumesConnectedPendingNodes(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume connected"}) if err != nil { t.Fatalf("create project: %v", err) } nodes := []domain.WorkflowNode{ {ID: "source-a", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"a\"}"}}}}, {ID: "source-b", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"b\"}"}}}}, {ID: "source-c", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"c\"}"}}}}, } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "无边批次恢复", Description: "resume connected", Type: domain.WorkflowTypeBatch, Nodes: nodes, Edges: []domain.WorkflowEdge{ {ID: "source-a-to-source-b", Source: "source-a", SourceHandle: "out-1", Target: "source-b", TargetHandle: "in-1"}, {ID: "source-b-to-source-c", Source: "source-b", SourceHandle: "out-1", Target: "source-c", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } startedAt := nowISO() run.Status = domain.RunStatusQueued run.CurrentNodeID = "" run.NodeExecutions = []domain.NodeExecutionRecord{{ NodeID: "source-a", NodeLabel: "真实数据源", Status: domain.NodeExecutionStatusCompleted, StartedAt: startedAt, EndedAt: startedAt, Artifacts: []domain.ExecutionArtifact{{ID: "artifact-a", RunID: run.ID, NodeID: "source-a", PortID: "out-1", Name: "a.json", Type: domain.PortTypeJSON, Data: map[string]any{"name": "a"}}}, }} if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) if len(completed.NodeExecutions) != 3 { t.Fatalf("expected all unconnected nodes executed, got %#v", completed.NodeExecutions) } for _, node := range nodes { if execution := mustNodeExecution(t, completed, node.ID); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected %s completed, got %s", node.ID, execution.Status) } } } func TestRuntimeReconcileActiveRunsRecoversSucceededNodeAndResumesDownstream(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover success"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "成功节点恢复", Description: "recover success", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}}, }, }, { ID: "annotate", Type: "operator", Data: domain.WorkflowNodeData{ Label: "视频标注", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "MODEL", Value: "stub"}}, }, }, { ID: "upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}}, }, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-annotate", Source: "source", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"}, {ID: "annotate-upload", Source: "annotate", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } sourceArtifact := domain.ExecutionArtifact{ ID: "artifact-source", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "payload.json", Type: domain.PortTypeJSON, Path: filepath.Join(t.TempDir(), "payload.json"), Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1}, } run.Status = domain.RunStatusRunning run.CurrentNodeID = "annotate" run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano) run.UpdatedAt = run.StartedAt run.NodeExecutions = []domain.NodeExecutionRecord{ { NodeID: "source", NodeLabel: "真实数据源", Status: domain.NodeExecutionStatusCompleted, StartedAt: run.StartedAt, EndedAt: run.StartedAt, Logs: []domain.LogEntryRecord{}, Artifacts: []domain.ExecutionArtifact{sourceArtifact}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, }, { NodeID: "annotate", NodeLabel: "视频标注", Status: domain.NodeExecutionStatusRunning, StartedAt: run.StartedAt, Logs: []domain.LogEntryRecord{{Timestamp: run.StartedAt, Level: "INFO", Message: "Starting node execution", Operator: "视频标注"}}, Artifacts: []domain.ExecutionArtifact{}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, }, } if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } launcher := testLauncher{ launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { env := append([]string{}, os.Environ()...) keys := make([]string, 0, len(input.Env)) for key := range input.Env { keys = append(keys, key) } sort.Strings(keys) env = append(env, "RUN_ID="+input.RunID, "WORKFLOW_ID="+input.WorkflowID, "NODE_ID="+input.NodeID, "NODE_LABEL="+input.NodeLabel) for _, key := range keys { env = append(env, key+"="+input.Env[key]) } cmd := exec.CommandContext(ctx, "python3", operatorRunnerPath(t)) cmd.Env = env output, err := cmd.CombinedOutput() if err != nil { return nil, fmt.Errorf("%w: %s", err, string(output)) } return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID), Namespace: "droboticflow"}, Output: string(output), }, nil }, inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) { if runID == run.ID && nodeID == "annotate" { return &k8s.JobInspection{ Reference: &k8s.JobReference{JobName: "job-annotate", Namespace: "droboticflow"}, State: k8s.JobStateSucceeded, Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"annotation-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"annotate\",\"portId\":\"out-1\",\"name\":\"annotation.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/annotation.json\",\"data\":{\"items\":[{\"label\":\"walking\"}],\"count\":1}}],\"logs\":[],\"resourceSeries\":[]}", }, nil } return &k8s.JobInspection{State: k8s.JobStateMissing}, nil }, } runtime := newTestRuntimeWithLauncher(t, dataStore, launcher) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile active runs: %v", err) } recovered, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get recovered run: %v", err) } if recovered == nil || recovered.Status != domain.RunStatusQueued { t.Fatalf("expected recovered run queued, got %#v", recovered) } if execution := mustNodeExecution(t, recovered, "annotate"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected annotate completed after recovery, got %s", execution.Status) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once after recovery: %v", err) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected downstream upload completed, got %s", execution.Status) } } func TestRuntimeReconcileActiveRunsCompletesRunWhenRecoveredFinalNodeSucceeded(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复完成项目", Description: "recover final success"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "最终节点恢复", Description: "recover final success", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, }, }, { ID: "sample", Type: "operator", Data: domain.WorkflowNodeData{ Label: "抽检任务", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}}, }, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-sample", Source: "source", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } startedAt := time.Now().UTC().Add(-3 * time.Minute).Format(time.RFC3339Nano) run.Status = domain.RunStatusRunning run.CurrentNodeID = "sample" run.StartedAt = startedAt run.UpdatedAt = startedAt run.NodeExecutions = []domain.NodeExecutionRecord{ { NodeID: "source", NodeLabel: "真实数据源", Status: domain.NodeExecutionStatusCompleted, StartedAt: startedAt, EndedAt: startedAt, Artifacts: []domain.ExecutionArtifact{{ ID: "artifact-source", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "payload.json", Type: domain.PortTypeJSON, Data: map[string]any{"ok": true}, }}, }, { NodeID: "sample", NodeLabel: "抽检任务", Status: domain.NodeExecutionStatusRunning, StartedAt: startedAt, Logs: []domain.LogEntryRecord{{Timestamp: startedAt, Level: "INFO", Message: "Starting node execution", Operator: "抽检任务"}}, }, } if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{ inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) { if runID == run.ID && nodeID == "sample" { return &k8s.JobInspection{ Reference: &k8s.JobReference{JobName: "job-sample", Namespace: "droboticflow"}, Image: "registry.example/operator@sha256:" + strings.Repeat("d", 64), State: k8s.JobStateSucceeded, Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"sample-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"sample\",\"portId\":\"out-1\",\"name\":\"audit-samples.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/audit-samples.json\",\"data\":{\"sampledCount\":1}}],\"logs\":[],\"resourceSeries\":[]}", }, nil } return &k8s.JobInspection{State: k8s.JobStateMissing}, nil }, }) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile active runs: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get completed run: %v", err) } if completed == nil || completed.Status != domain.RunStatusCompleted { t.Fatalf("expected recovered final node to complete run, got %#v", completed) } if completed.CurrentNodeID != "" { t.Fatalf("expected current node cleared, got %q", completed.CurrentNodeID) } if execution := mustNodeExecution(t, completed, "sample"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected sample completed, got %s", execution.Status) } if execution := mustNodeExecution(t, completed, "sample"); execution.OperatorImageID != "registry.example/operator@sha256:"+strings.Repeat("d", 64) { t.Fatalf("recovery lost actual Job digest: %s", execution.OperatorImageID) } finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) if err != nil { t.Fatalf("get workflow: %v", err) } if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted { t.Fatalf("expected workflow completed, got %#v", finalWorkflow) } } func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() settings, err := dataStore.GetSettings(ctx) if err != nil { t.Fatalf("get settings: %v", err) } settings.Worker.MaxConcurrentRuns = 3 if err := dataStore.UpsertSettings(ctx, *settings); err != nil { t.Fatalf("upsert settings: %v", err) } project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Parallel DAG 项目", Description: "parallel dag"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "并行分支工作流", Description: "parallel dag", Type: domain.WorkflowTypeBatch, MaxConcurrentRuns: 3, Nodes: []domain.WorkflowNode{ { ID: "source", Type: "operator", Data: domain.WorkflowNodeData{ Label: "OSS 数据源", Category: "IO", Description: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "parallel"}, }, }, Position: domain.Position{X: 0, Y: 0}, }, { ID: "branch-a", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "branch a", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-a", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"a\"}' > result-a.json"}}, }, Position: domain.Position{X: 200, Y: -120}, }, { ID: "branch-b", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "branch b", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-b", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"b\"}' > result-b.json"}}, }, Position: domain.Position{X: 200, Y: 0}, }, { ID: "branch-c", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "branch c", Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-c", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"c\"}' > result-c.json"}}, }, Position: domain.Position{X: 200, Y: 120}, }, }, Edges: []domain.WorkflowEdge{ {ID: "source-a", Source: "source", SourceHandle: "out-1", Target: "branch-a", TargetHandle: "in-1"}, {ID: "source-b", Source: "source", SourceHandle: "out-1", Target: "branch-b", TargetHandle: "in-1"}, {ID: "source-c", Source: "source", SourceHandle: "out-1", Target: "branch-c", TargetHandle: "in-1"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } runningRun := waitForRunningNodeCount(t, ctx, dataStore, run.ID, 3, 5*time.Second) if runningRun.Status != domain.RunStatusRunning { t.Fatalf("expected run to be running while branches execute, got %s", runningRun.Status) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second) for _, nodeID := range []string{"branch-a", "branch-b", "branch-c"} { if execution := mustNodeExecution(t, completed, nodeID); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected node %s to complete, got %s", nodeID, execution.Status) } } } func TestRuntimeProcessRunContinuesConnectedBranchesPastConcurrencyLimit(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() settings, err := dataStore.GetSettings(ctx) if err != nil { t.Fatalf("get settings: %v", err) } settings.Worker.MaxConcurrentRuns = 2 if err := dataStore.UpsertSettings(ctx, *settings); err != nil { t.Fatalf("upsert settings: %v", err) } project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Disconnected 项目", Description: "disconnected nodes"}) if err != nil { t.Fatalf("create project: %v", err) } nodes := []domain.WorkflowNode{{ ID: "root", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "root node", Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"node":"root"}' > root.json`}}, }, }} edges := make([]domain.WorkflowEdge, 0, 5) for index := 0; index < 5; index++ { nodeID := fmt.Sprintf("independent-%d", index) nodes = append(nodes, domain.WorkflowNode{ ID: nodeID, Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "independent node", Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: fmt.Sprintf(`printf '{"node":"%s"}' > %s.json`, nodeID, nodeID)}}, }, }) edges = append(edges, domain.WorkflowEdge{ID: "root-to-" + nodeID, Source: "root", Target: nodeID}) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "单连通分支工作流", Description: "connected branch batch", Type: domain.WorkflowTypeBatch, MaxConcurrentRuns: 2, Nodes: nodes, Edges: edges, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second) if len(completed.NodeExecutions) != len(nodes) { t.Fatalf("expected %d executed nodes, got %d: %#v", len(nodes), len(completed.NodeExecutions), completed.NodeExecutions) } for _, node := range nodes { if execution := mustNodeExecution(t, completed, node.ID); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected node %s to complete, got %s", node.ID, execution.Status) } } } func TestRuntimeRerunSubtreeOnlyExecutesSelectedSubgraph(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "子链路重跑项目", Description: "rerun subtree"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "子链路重跑工作流", Description: "rerun subtree", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "root", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "FLOW", Description: "root", Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"root":true}' > root.json`}}, }, }, { ID: "start-a", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "FLOW", Description: "branch a start", Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a"}' > branch-a.json`}}, }, }, { ID: "downstream-a", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "FLOW", Description: "branch a end", Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a-downstream"}' > branch-a-downstream.json`}}, }, }, { ID: "start-b", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "FLOW", Description: "branch b start", Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"b"}' > branch-b.json`}}, }, }, }, Edges: []domain.WorkflowEdge{ {ID: "root-to-start-a", Source: "root", Target: "start-a"}, {ID: "root-to-start-b", Source: "root", Target: "start-b"}, {ID: "a-to-downstream", Source: "start-a", Target: "downstream-a"}, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } run.Trigger = domain.RunTriggerRerunSubtree run.RestartFromNodeID = "start-a" startedAt := nowISO() run.NodeExecutions = []domain.NodeExecutionRecord{{ NodeID: "root", NodeLabel: "Shell 命令执行", Status: domain.NodeExecutionStatusCompleted, StartedAt: startedAt, EndedAt: startedAt, Artifacts: []domain.ExecutionArtifact{{ID: "artifact-root", RunID: run.ID, NodeID: "root", PortID: "out-1", Name: "root.json", Type: domain.PortTypeJSON, Data: map[string]any{"root": true}}}, }} run.Status = domain.RunStatusQueued run.UpdatedAt = nowISO() if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert rerun subtree run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second) if execution := mustNodeExecution(t, completed, "start-a"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected start-a to complete, got %s", execution.Status) } if execution := mustNodeExecution(t, completed, "downstream-a"); execution.Status != domain.NodeExecutionStatusCompleted { t.Fatalf("expected downstream-a to complete, got %s", execution.Status) } for _, execution := range completed.NodeExecutions { if execution.NodeID == "start-b" { t.Fatalf("expected unrelated branch start-b not to execute, got %#v", execution) } } } func TestProcessRunKeepsClaimedRunRunningWhenExistingNodeExecutionIsActive(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() runtime := &Runtime{ store: dataStore, operatorRunnerImage: "runner:latest", workspaceRoot: "/workspace", launcher: testLauncher{}, } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: "", Name: "resume-active-node", Description: "resume active execution", Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source"}}}, Edges: []domain.WorkflowEdge{}, Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard, Schedule: nil, ScanConfig: nil, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } run.Status = domain.RunStatusQueued run.CurrentNodeID = "source" run.NodeExecutions = []domain.NodeExecutionRecord{ { NodeID: "source", NodeLabel: "Source", Status: domain.NodeExecutionStatusRunning, StartedAt: nowISO(), }, } run.UpdatedAt = nowISO() if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } processed := mustProcessOnceRun(t, runtime, ctx) if processed.Status != domain.RunStatusRunning { t.Fatalf("expected running status, got %s", processed.Status) } if processed.CurrentNodeID != "source" { t.Fatalf("expected currentNodeID=source, got %s", processed.CurrentNodeID) } if execution := mustNodeExecution(t, processed, "source"); execution.Status != domain.NodeExecutionStatusRunning { t.Fatalf("expected source execution running, got %s", execution.Status) } } func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Timeout 项目", Description: "timeout"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "超时工作流", Description: "timeout", Type: domain.WorkflowTypeBatch, RunTimeoutSeconds: 1, Nodes: []domain.WorkflowNode{ { ID: "slow-node", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "slow", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}}, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } failed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get failed run: %v", err) } if failed == nil || failed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %#v", failed) } if !strings.Contains(failed.Error, "超时") { t.Fatalf("expected timeout error, got %q", failed.Error) } execution := mustNodeExecution(t, failed, "slow-node") if execution.Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected failed node execution, got %s", execution.Status) } storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) if err != nil { t.Fatalf("get workflow: %v", err) } if storedWorkflow == nil || storedWorkflow.Status != domain.WorkflowStatusFailed { t.Fatalf("expected failed workflow, got %#v", storedWorkflow) } } func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() markerPath := filepath.Join(t.TempDir(), "retry-marker") project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry 项目", Description: "retry"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "自动重试工作流", Description: "retry", Type: domain.WorkflowTypeBatch, MaxRetryAttempts: 1, Nodes: []domain.WorkflowNode{ { ID: "flaky-node", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "flaky", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{ Key: "COMMAND", Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json", }}, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create initial run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("first process once: %v", err) } firstAttempt, err := dataStore.GetRun(ctx, initialRun.ID) if err != nil { t.Fatalf("get first attempt: %v", err) } if firstAttempt == nil || firstAttempt.Status != domain.RunStatusFailed { t.Fatalf("expected first attempt to fail, got %#v", firstAttempt) } if firstAttempt.Attempt != 1 { t.Fatalf("expected first attempt number to be 1, got %d", firstAttempt.Attempt) } runs, err := dataStore.ListRuns(ctx, "") if err != nil { t.Fatalf("list runs: %v", err) } if len(runs) != 2 { t.Fatalf("expected original + retry run, got %d", len(runs)) } var retryRun domain.RunRecord foundRetry := false for _, candidate := range runs { if candidate.ID == initialRun.ID { continue } retryRun = candidate foundRetry = true break } if !foundRetry { t.Fatalf("expected retry run to be created") } if retryRun.Trigger != domain.RunTriggerRetry { t.Fatalf("expected retry trigger, got %s", retryRun.Trigger) } if retryRun.Attempt != 2 { t.Fatalf("expected retry attempt number to be 2, got %d", retryRun.Attempt) } if retryRun.ParentRunID != initialRun.ID { t.Fatalf("expected retry parent run %s, got %s", initialRun.ID, retryRun.ParentRunID) } if retryRun.Status != domain.RunStatusQueued { t.Fatalf("expected retry run to be queued, got %s", retryRun.Status) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("second process once: %v", err) } completedRetry, err := dataStore.GetRun(ctx, retryRun.ID) if err != nil { t.Fatalf("get retry run: %v", err) } if completedRetry == nil || completedRetry.Status != domain.RunStatusCompleted { t.Fatalf("expected retry run to complete, got %#v", completedRetry) } finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) if err != nil { t.Fatalf("get final workflow: %v", err) } if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted { t.Fatalf("expected workflow to recover and complete, got %#v", finalWorkflow) } } func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Node Timeout 项目", Description: "node timeout"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "节点超时工作流", Description: "node timeout", Type: domain.WorkflowTypeBatch, NodeTimeoutSeconds: 1, Nodes: []domain.WorkflowNode{ { ID: "slow-node", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "slow", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}}, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once: %v", err) } failed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get failed run: %v", err) } if failed == nil || failed.Status != domain.RunStatusFailed { t.Fatalf("expected failed run, got %#v", failed) } if !strings.Contains(failed.Error, "节点执行超时") { t.Fatalf("expected node timeout error, got %q", failed.Error) } } func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() markerPath := filepath.Join(t.TempDir(), "retry-backoff-marker") project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry Backoff 项目", Description: "retry backoff"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "重试延迟工作流", Description: "retry backoff", Type: domain.WorkflowTypeBatch, MaxRetryAttempts: 1, RetryBackoffSeconds: 2, Nodes: []domain.WorkflowNode{ { ID: "flaky-node", Type: "operator", Data: domain.WorkflowNodeData{ Label: "Shell 命令执行", Category: "UTIL", Description: "flaky", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}}, Hyperparams: []domain.HyperParam{{ Key: "COMMAND", Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json", }}, }, Position: domain.Position{X: 0, Y: 0}, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create initial run: %v", err) } runtime, err := newTestRuntime(t, dataStore) if err != nil { t.Fatalf("create runtime: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("first process once: %v", err) } runs, err := dataStore.ListRuns(ctx, "") if err != nil { t.Fatalf("list runs: %v", err) } if len(runs) != 2 { t.Fatalf("expected original + retry run, got %d", len(runs)) } var retryRun domain.RunRecord for _, candidate := range runs { if candidate.ID != initialRun.ID { retryRun = candidate break } } if retryRun.AvailableAt == "" { t.Fatalf("expected retry run to carry availableAt") } if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available before backoff expires: %v", err) } stillQueued, err := dataStore.GetRun(ctx, retryRun.ID) if err != nil { t.Fatalf("get retry run before backoff: %v", err) } if stillQueued == nil || stillQueued.Status != domain.RunStatusQueued { t.Fatalf("expected retry run to remain queued before backoff expires, got %#v", stillQueued) } time.Sleep(2200 * time.Millisecond) if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available after backoff expires: %v", err) } completedRetry := waitForRunStatus(t, ctx, dataStore, retryRun.ID, domain.RunStatusCompleted, 8*time.Second) if completedRetry.Attempt != 2 { t.Fatalf("expected retry attempt number to stay 2, got %d", completedRetry.Attempt) } } func newReconcileFixture(t *testing.T, dataStore store.Store, name string) (*domain.WorkflowRecord, *domain.RunRecord) { t.Helper() ctx := context.Background() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: name + " project", Description: "reconcile"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: name + " workflow", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ ID: "node-1", Type: "operator", Data: domain.WorkflowNodeData{Label: name + " node"}, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } startedAt := time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano) run.Status = domain.RunStatusRunning run.CurrentNodeID = "node-1" run.StartedAt = startedAt run.UpdatedAt = startedAt run.NodeExecutions = []domain.NodeExecutionRecord{{ NodeID: "node-1", NodeLabel: name + " node", Attempt: 1, Status: domain.NodeExecutionStatusRunning, StartedAt: startedAt, }} if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert run: %v", err) } return workflow, run } func TestRuntimeReconcileDefersToAnUnexpiredExecutionLease(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, run := newReconcileFixture(t, dataStore, "leased execution") run.ExecutionLeaseOwner = "worker-with-active-launch" run.ExecutionLeaseExpiresAt = time.Now().UTC().Add(10 * time.Minute).Format(time.RFC3339Nano) if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert leased run: %v", err) } inspectCalls := 0 runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) { inspectCalls++ return &k8s.JobInspection{State: k8s.JobStateMissing}, nil }}) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile leased run: %v", err) } stored, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get leased run: %v", err) } if inspectCalls != 0 { t.Fatalf("expected active lease to suppress concurrent Job inspection, got %d calls", inspectCalls) } if stored.Status != domain.RunStatusRunning || mustNodeExecution(t, stored, "node-1").Status != domain.NodeExecutionStatusRunning { t.Fatalf("expected leased execution to remain untouched, got %#v", stored) } } func TestRuntimeReconcileFailedJobTerminalizesRunAndNode(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() workflow, run := newReconcileFixture(t, dataStore, "failed job") runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) { return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "container exited with code 2"}, nil }}) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile active runs: %v", err) } failed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "code 2") { t.Fatalf("expected failed run with diagnostics, got %#v", failed) } if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_failed" || failed.Failure.NodeID != "node-1" { t.Fatalf("expected structured kubernetes failure, got %#v", failed.Failure) } if execution := mustNodeExecution(t, failed, "node-1"); execution.Status != domain.NodeExecutionStatusFailed { t.Fatalf("expected failed node execution, got %#v", execution) } storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) if err != nil || storedWorkflow.Status != domain.WorkflowStatusFailed { t.Fatalf("expected failed workflow, got %#v err=%v", storedWorkflow, err) } } func TestRuntimeReconcileSucceededJobFailsOrphanRunWhenWorkflowWasDeleted(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, run := newReconcileFixture(t, dataStore, "deleted workflow") run.WorkflowID = "workflow-deleted-during-run" if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("orphan active run: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) { return &k8s.JobInspection{State: k8s.JobStateSucceeded}, nil }}) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile orphan run: %v", err) } failed, err := dataStore.GetRun(ctx, run.ID) if err != nil || failed == nil { t.Fatalf("get orphan run: run=%#v err=%v", failed, err) } if failed.Status != domain.RunStatusFailed || failed.Failure == nil || failed.Failure.Code != "workflow_not_found" { t.Fatalf("expected orphan run to fail terminally, got %#v", failed) } } func TestRuntimeReconcileStuckImagePullTerminalizesButHealthyLongJobStaysRunning(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, blockedRun := newReconcileFixture(t, dataStore, "blocked image") _, healthyRun := newReconcileFixture(t, dataStore, "healthy training") runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) { if runID == blockedRun.ID { return &k8s.JobInspection{ State: k8s.JobStatePending, CreatedAt: time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano), BlockingReason: "ImagePullBackOff", BlockingMessage: "back-off pulling image", }, nil } return &k8s.JobInspection{State: k8s.JobStateActive, CreatedAt: time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339Nano)}, nil }}) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("reconcile active runs: %v", err) } blocked, _ := dataStore.GetRun(ctx, blockedRun.ID) healthy, _ := dataStore.GetRun(ctx, healthyRun.ID) if blocked.Status != domain.RunStatusFailed || !strings.Contains(blocked.Error, "ImagePullBackOff") { t.Fatalf("expected blocked image run to fail explicitly, got %#v", blocked) } if blocked.Failure == nil || blocked.Failure.Code != "kubernetes_image_pull_blocked" || blocked.Failure.Stage != "scheduling" { t.Fatalf("expected structured image pull failure, got %#v", blocked.Failure) } if healthy.Status != domain.RunStatusRunning { t.Fatalf("expected healthy long-running job unchanged, got %#v", healthy) } } func TestRuntimeReconcileMissingJobFailsAfterRecoveryLimit(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, run := newReconcileFixture(t, dataStore, "missing job") runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) runtime.missingMaxRecoveries = 1 if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("first reconcile: %v", err) } recovered, _ := dataStore.GetRun(ctx, run.ID) if recovered.Status != domain.RunStatusQueued || !strings.Contains(mustNodeExecution(t, recovered, "node-1").Error, "recovery attempt 1/1") { t.Fatalf("expected first recovery to requeue, got %#v", recovered) } recovered.Status = domain.RunStatusRunning recovered.CurrentNodeID = "node-1" recovered.UpdatedAt = time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano) execution := mustNodeExecution(t, recovered, "node-1") execution.Status = domain.NodeExecutionStatusRunning execution.StartedAt = recovered.UpdatedAt recovered.NodeExecutions = []domain.NodeExecutionRecord{execution} if err := dataStore.UpsertRun(ctx, *recovered); err != nil { t.Fatalf("simulate recovered execution: %v", err) } if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("second reconcile: %v", err) } failed, _ := dataStore.GetRun(ctx, run.ID) if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "after 1 recovery attempts") { t.Fatalf("expected missing job to fail after recovery limit, got %#v", failed) } if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_missing" { t.Fatalf("expected structured missing-job failure, got %#v", failed.Failure) } } func TestRuntimeLaunchMissingJobRequeuesAndUsesNextNodeAttempt(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "missing launch", Description: "recover active launch"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "missing launch workflow", Description: "recover active launch", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ ID: "train", Type: "operator", Data: domain.WorkflowNodeData{Label: "train"}, }}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } launchAttempts := make([]int, 0, 2) runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(_ context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { launchAttempts = append(launchAttempts, input.Attempt) if len(launchAttempts) == 1 { return nil, &k8s.JobNotFoundError{JobName: input.RunID + "-" + input.NodeID + "-a1"} } if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" { if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil { return nil, err } } return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: input.RunID + "-" + input.NodeID + "-a2", Namespace: "droboticflow"}, Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, nil }}) if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("first process once: %v", err) } requeued, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get requeued run: %v", err) } if requeued.Status != domain.RunStatusQueued || requeued.Failure != nil { t.Fatalf("expected missing Job to be requeued without terminal failure, got %#v", requeued) } firstExecution := mustNodeExecution(t, requeued, "train") if firstExecution.Status != domain.NodeExecutionStatusFailed || !strings.Contains(firstExecution.Error, "recovery attempt 1/3") { t.Fatalf("expected bounded missing-Job recovery evidence, got %#v", firstExecution) } requeued.AvailableAt = nowISO() if err := dataStore.UpsertRun(ctx, *requeued); err != nil { t.Fatalf("make recovered run available: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("second process once: %v", err) } completed, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get completed run: %v", err) } if completed.Status != domain.RunStatusCompleted { t.Fatalf("expected recovered run to complete, got %#v", completed) } if len(launchAttempts) != 2 || launchAttempts[0] != 1 || launchAttempts[1] != 2 { t.Fatalf("expected Kubernetes attempts [1 2], got %#v", launchAttempts) } if len(completed.K8sRefs) != 1 || completed.K8sRefs[0].Attempt != 2 || !strings.HasSuffix(completed.K8sRefs[0].JobName, "-a2") { t.Fatalf("expected recovered Job reference to retain attempt 2, got %#v", completed.K8sRefs) } } func TestRuntimeReconcileIsolatesPerRunInspectionErrors(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, brokenRun := newReconcileFixture(t, dataStore, "inspection error") _, failedRun := newReconcileFixture(t, dataStore, "independent failure") runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) { if runID == brokenRun.ID { return nil, fmt.Errorf("temporary kubernetes error") } return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "independent failure"}, nil }}) if err := runtime.reconcileActiveRuns(ctx); err != nil { t.Fatalf("per-run error must not fail the cycle: %v", err) } broken, _ := dataStore.GetRun(ctx, brokenRun.ID) failed, _ := dataStore.GetRun(ctx, failedRun.ID) if broken.Status != domain.RunStatusRunning { t.Fatalf("expected errored inspection unchanged, got %#v", broken) } if failed.Status != domain.RunStatusFailed { t.Fatalf("expected independent run reconciled, got %#v", failed) } } func TestRuntimeLeaseOwnerChangeCancelsExecutionContext(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, run := newReconcileFixture(t, dataStore, "lease loss") run.ExecutionLeaseOwner = "other-worker" run.ExecutionLeaseExpiresAt = time.Now().UTC().Add(time.Hour).Format(time.RFC3339Nano) if err := dataStore.UpsertRun(ctx, *run); err != nil { t.Fatalf("upsert foreign lease: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) runtime.dispatchOwner = "this-worker" runtime.runLeaseTTL = 30 * time.Millisecond leaseCtx, cancelLease := context.WithCancelCause(ctx) defer cancelLease(context.Canceled) go runtime.renewRunLeaseUntil(leaseCtx, run.ID, cancelLease) deadline := time.After(time.Second) for !errors.Is(context.Cause(leaseCtx), errRunLeaseLost) { select { case <-deadline: t.Fatalf("expected lease owner change to cancel execution context, cause=%v", context.Cause(leaseCtx)) case <-time.After(5 * time.Millisecond): } } } func TestRuntimeProcessAvailableAppliesPersistedPollInterval(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() settings, err := dataStore.GetSettings(ctx) if err != nil { t.Fatalf("get settings: %v", err) } settings.Worker.PollIntervalMS = 750 if err := dataStore.UpsertSettings(ctx, *settings); err != nil { t.Fatalf("upsert settings: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) runtime.SetConfiguredPollInterval(2 * time.Second) if err := runtime.ProcessAvailable(ctx); err != nil { t.Fatalf("process available: %v", err) } if got := runtime.EffectivePollInterval(); got != 750*time.Millisecond { t.Fatalf("expected persisted 750ms poll interval, got %s", got) } } func TestLatestStructuredProgressUsesLatestValidMarker(t *testing.T) { logs := strings.Join([]string{ "ordinary training output", `DROBOTICFLOW_PROGRESS {"stage":"train","epoch":1,"step":20,"maxStep":100,"samplesPerSecond":4.5,"estimatedRemainingSeconds":18,"checkpoint":"checkpoint-20","updatedAt":"2026-07-10T08:00:00Z"}`, "DROBOTICFLOW_PROGRESS {partial", `prefix DROBOTICFLOW_PROGRESS {"stage":"train","epoch":2,"step":40,"maxStep":100,"samplesPerSecond":5.5,"estimatedRemainingSeconds":11,"checkpoint":"checkpoint-40","updatedAt":"2026-07-10T08:01:00Z"}`, }, "\n") progress, err := latestStructuredProgress(logs) if err != nil { t.Fatalf("parse progress: %v", err) } if progress == nil || progress.Epoch != 2 || progress.Step != 40 || progress.Checkpoint != "checkpoint-40" { t.Fatalf("expected latest valid marker, got %#v", progress) } } func TestNodeProgressReporterPersistsStructuredMarker(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() _, run := newReconcileFixture(t, dataStore, "structured progress") runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{tailNodePodLogs: func(context.Context, string, string, int64) (string, error) { return `DROBOTICFLOW_PROGRESS {"stage":"train","epoch":3,"step":75,"maxStep":100,"samplesPerSecond":6.25,"estimatedRemainingSeconds":4,"checkpoint":"checkpoint-75"}`, nil }}) reporterCtx, cancelReporter := context.WithCancel(ctx) done := make(chan struct{}) go runtime.runNodeProgressReporterWithInterval(reporterCtx, run.ID, "node-1", false, 5*time.Millisecond, func() { close(done) }) deadline := time.Now().Add(time.Second) for { stored, err := dataStore.GetRun(ctx, run.ID) if err != nil { t.Fatalf("get run: %v", err) } if stored.Progress != nil { if stored.Progress.Step != 75 || stored.Progress.Checkpoint != "checkpoint-75" || stored.Progress.UpdatedAt == "" { t.Fatalf("unexpected structured progress: %#v", stored.Progress) } break } if time.Now().After(deadline) { t.Fatal("timed out waiting for structured progress") } time.Sleep(5 * time.Millisecond) } cancelReporter() select { case <-done: case <-time.After(time.Second): t.Fatal("progress reporter did not stop") } } func TestSyncStopMetadataPreservesAtomicProgressUpdates(t *testing.T) { target := &domain.RunRecord{} source := &domain.RunRecord{Progress: &domain.RunProgress{Stage: "train", Step: 9, UpdatedAt: "2026-07-10T08:00:00Z"}} syncStopMetadata(target, source) if target.Progress == nil || target.Progress.Step != 9 { t.Fatalf("expected progress copied from persisted run, got %#v", target.Progress) } source.Progress.Step = 10 if target.Progress.Step != 9 { t.Fatalf("expected progress copy to avoid pointer aliasing, got %#v", target.Progress) } } func TestCompletedNodePersistsTerminalStructuredProgressMarker(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "terminal progress", Description: "progress"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "terminal progress", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{{ID: "train", Type: "operator", Data: domain.WorkflowNodeData{Label: "train"}}}, }) if err != nil { t.Fatalf("create workflow: %v", err) } run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) if err != nil { t.Fatalf("create run: %v", err) } runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(_ context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { stageManifestPath := input.Env["STAGE_MANIFEST_PATH"] if err := os.MkdirAll(filepath.Dir(stageManifestPath), 0o755); err != nil { return nil, err } if err := os.WriteFile(stageManifestPath, []byte(`{"dataRefs":[]}`), 0o644); err != nil { return nil, err } return &k8s.LaunchResult{ Reference: &k8s.JobReference{JobName: "terminal-progress", Namespace: "droboticflow"}, Output: strings.Join([]string{ `DROBOTICFLOW_PROGRESS {"stage":"completed","epoch":4,"step":100,"maxStep":100,"samplesPerSecond":6.5,"estimatedRemainingSeconds":0,"checkpoint":"checkpoint-final"}`, `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, }, "\n"), }, nil }}) result, err := runtime.ProcessOnce(ctx) if err != nil { t.Fatalf("process run: %v", err) } if result == nil || result.Run == nil || result.Run.ID != run.ID { t.Fatalf("unexpected process result: %#v", result) } completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, time.Second) if completed.Progress == nil || completed.Progress.Stage != "completed" || completed.Progress.Step != 100 || completed.Progress.Checkpoint != "checkpoint-final" { t.Fatalf("expected terminal structured progress, got %#v", completed.Progress) } } // shouldReportNodeProgress must flag only custom-container operators — builtin // operator-runner pods self-report resourceSeries via their in-process ResourceSampler, // so platform-side GPU sampling is reserved for nodes whose resolved image differs from // the operator-runner image (mirroring k8s.Launch's shouldSample gate). func TestShouldReportNodeProgressOnlyForCustomContainers(t *testing.T) { rt := &Runtime{operatorRunnerImage: "droboticflow-operator-runner:latest"} cases := []struct { name string node domain.WorkflowNode want bool }{ {"custom image", domain.WorkflowNode{Data: domain.WorkflowNodeData{Image: "sam3-video-tracker:latest"}}, true}, {"builtin operator-runner image", domain.WorkflowNode{Data: domain.WorkflowNodeData{Image: "droboticflow-operator-runner:latest"}}, false}, {"digest-pinned custom image", domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorImageID: "sam3-video-tracker@sha256:40efc6ba77e0bfb3a029825b20a36615445f7c62a0279804971c543c46ccd08f"}}, true}, {"no image falls back to operator-runner", domain.WorkflowNode{Data: domain.WorkflowNodeData{}}, false}, } for _, tc := range cases { if got := shouldReportNodeProgress(rt, tc.node); got != tc.want { t.Errorf("%s: got %v want %v", tc.name, got, tc.want) } } } // mapGPUSample must round-trip every GPU-relevant field the launcher reports onto the // resource monitor's domain.ResourceSample (the same mapping the executor uses when // folding the launcher's final series). func TestMapGPUSampleCopiesFields(t *testing.T) { in := k8s.GPUSample{ Timestamp: "2026-07-10T00:00:00.000000Z", GPU: 42, GPUMemoryUsedMiB: 28000, GPUMemoryTotalMiB: 81920, GPUName: "NVIDIA A800-SXM4-80GB", GPUCount: 1, } out := mapGPUSample(in) if out.Timestamp != in.Timestamp || out.GPU != in.GPU || out.GPUMemoryUsedMiB != in.GPUMemoryUsedMiB || out.GPUMemoryTotalMiB != in.GPUMemoryTotalMiB || out.GPUName != in.GPUName || out.GPUCount != in.GPUCount { t.Fatalf("mapGPUSample mismatch: in=%#v out=%#v", in, out) } }