5513 lines
203 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package worker
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/store"
)
type testLauncher struct {
launch func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error)
inspect func(context.Context, string, string) (*k8s.JobInspection, error)
gpuCapacity func(context.Context) (*k8s.GPUCapacitySnapshot, error)
terminateRun func(context.Context, string, string) error
samplePodGPUForRun func(context.Context, string, string) (k8s.GPUSample, error)
tailNodePodLogs func(context.Context, string, string, int64) (string, error)
}
func (launcher testLauncher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
if launcher.tailNodePodLogs == nil {
return "", fmt.Errorf("no pod logs for run %s node %s", runID, nodeID)
}
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines)
}
func (launcher testLauncher) Launch(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
if launcher.launch == nil {
return nil, fmt.Errorf("test launcher launch not configured")
}
return launcher.launch(ctx, input)
}
func (launcher testLauncher) Inspect(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) {
if launcher.inspect == nil {
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
}
return launcher.inspect(ctx, runID, nodeID)
}
func (launcher testLauncher) TerminateRun(ctx context.Context, namespace string, runID string) error {
if launcher.terminateRun == nil {
return nil
}
return launcher.terminateRun(ctx, namespace, runID)
}
func (launcher testLauncher) SamplePodGPUForRun(ctx context.Context, runID string, nodeID string) (k8s.GPUSample, error) {
if launcher.samplePodGPUForRun == nil {
return k8s.GPUSample{}, fmt.Errorf("no running pod for run %s node %s", runID, nodeID)
}
return launcher.samplePodGPUForRun(ctx, runID, nodeID)
}
func (launcher testLauncher) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) {
if launcher.gpuCapacity == nil {
return &k8s.GPUCapacitySnapshot{ResourceName: "nvidia.com/gpu", SelectorSummary: "<none>"}, nil
}
return launcher.gpuCapacity(ctx)
}
func operatorRunnerPath(t *testing.T) string {
t.Helper()
path, err := filepath.Abs("../../backend/operator-runner/main.py")
if err != nil {
t.Fatalf("resolve operator runner: %v", err)
}
return path
}
func newTestRuntime(t *testing.T, dataStore store.Store) (*Runtime, error) {
t.Helper()
t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "true")
runnerPath := operatorRunnerPath(t)
return &Runtime{
store: dataStore,
operatorRunnerImage: "operator-runner:latest",
workspaceRoot: t.TempDir(),
launcher: testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
env := append([]string{}, os.Environ()...)
keys := make([]string, 0, len(input.Env))
for key := range input.Env {
keys = append(keys, key)
}
sort.Strings(keys)
env = append(env,
"RUN_ID="+input.RunID,
"WORKFLOW_ID="+input.WorkflowID,
"NODE_ID="+input.NodeID,
"NODE_LABEL="+input.NodeLabel,
)
for _, key := range keys {
env = append(env, key+"="+input.Env[key])
}
cmd := exec.CommandContext(ctx, "python3", runnerPath)
cmd.Env = env
output, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("%w: %s", err, string(output))
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{
JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID),
Namespace: "droboticflow",
},
Output: string(output),
}, nil
},
},
}, nil
}
func newTestRuntimeWithLauncher(t *testing.T, dataStore store.Store, launcher jobLauncher) *Runtime {
t.Helper()
t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "true")
return &Runtime{
store: dataStore,
operatorRunnerImage: "operator-runner:latest",
workspaceRoot: t.TempDir(),
launcher: launcher,
}
}
func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord {
t.Helper()
for _, item := range run.NodeExecutions {
if item.NodeID == nodeID {
return item
}
}
t.Fatalf("missing node execution for %s", nodeID)
return domain.NodeExecutionRecord{}
}
func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name string) domain.ExecutionArtifact {
t.Helper()
for _, item := range execution.Artifacts {
if item.Name == name {
return item
}
}
t.Fatalf("missing artifact %s on node %s", name, execution.NodeID)
return domain.ExecutionArtifact{}
}
func mustArtifactPayload(t *testing.T, artifact domain.ExecutionArtifact) map[string]any {
t.Helper()
if artifact.Data != nil {
if dataRef, ok := artifact.Data["dataRef"].(map[string]any); ok {
if payloadPath, ok := dataRef["payloadPath"].(string); ok && strings.TrimSpace(payloadPath) != "" {
payload, err := os.ReadFile(payloadPath)
if err != nil {
t.Fatalf("read artifact payload %s: %v", payloadPath, err)
}
var decoded map[string]any
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("decode artifact payload %s: %v", payloadPath, err)
}
return decoded
}
}
return artifact.Data
}
if strings.TrimSpace(artifact.Path) != "" {
payload, err := os.ReadFile(artifact.Path)
if err == nil {
var decoded map[string]any
if err := json.Unmarshal(payload, &decoded); err == nil {
return decoded
}
}
}
t.Fatalf("artifact %s has no structured payload", artifact.Name)
return nil
}
func mustProcessOnceRun(t *testing.T, runtime *Runtime, ctx context.Context) *domain.RunRecord {
t.Helper()
result, err := runtime.ProcessOnce(ctx)
if err != nil {
t.Fatalf("process run: %v", err)
}
if result == nil || result.Run == nil {
t.Fatalf("expected process-once to return a run, got %#v", result)
}
return result.Run
}
func TestProcessOnceResultJSONUsesLightweightRunSummary(t *testing.T) {
run := &domain.RunRecord{
ID: "run-1",
WorkflowID: "workflow-1",
WorkflowName: "workflow",
ProjectID: "project-1",
ProjectName: "project",
Status: domain.RunStatusCompleted,
Attempt: 2,
CurrentNodeID: "node-2",
ExecutionLeaseOwner: "worker-1",
ExecutionLeaseExpiresAt: "2026-07-02T00:00:00Z",
UpdatedAt: "2026-07-02T00:00:01Z",
Logs: []domain.LogEntryRecord{{
Level: "INFO",
Message: strings.Repeat("log-", 100),
}},
NodeExecutions: []domain.NodeExecutionRecord{{
NodeID: "node-1",
Status: domain.NodeExecutionStatusCompleted,
Artifacts: []domain.ExecutionArtifact{{
ID: "artifact-1",
Name: "large.json",
Data: map[string]any{"payload": strings.Repeat("data-", 100)},
}},
Logs: []domain.LogEntryRecord{{Message: strings.Repeat("node-log-", 100)}},
}},
Artifacts: []domain.ExecutionArtifact{{
ID: "run-artifact-1",
Name: "report.json",
Data: map[string]any{"payload": strings.Repeat("report-", 100)},
}},
}
result := newProcessOnceResult(ProcessOnceStatusClaimed, run)
if result.Run != run {
t.Fatalf("expected runtime result to retain the full run pointer")
}
encoded, err := json.Marshal(result)
if err != nil {
t.Fatalf("marshal process-once result: %v", err)
}
body := string(encoded)
for _, forbidden := range []string{"nodeExecutions", "logs", "resourceSeries", "k8sRefs", "large.json", "node-log-", "report-"} {
if strings.Contains(body, forbidden) {
t.Fatalf("process-once response leaked full run field %q in %s", forbidden, body)
}
}
for _, expected := range []string{`"status":"claimed"`, `"id":"run-1"`, `"nodeExecutionCount":1`, `"artifactCount":2`, `"currentNodeId":"node-2"`} {
if !strings.Contains(body, expected) {
t.Fatalf("process-once response missing %s in %s", expected, body)
}
}
}
func TestRuntimeProcessAvailableSkipsAlreadyActiveRun(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "active run", Description: "skip"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "workflow",
Description: "skip active run",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
Nodes: []domain.WorkflowNode{{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "source",
Outputs: []domain.PortDefinition{{
ID: "out-1",
Name: "payload",
Type: domain.PortTypeJSON,
}},
},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
launchCount := 0
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
launchCount++
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "job-" + input.RunID + "-" + input.NodeID, Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
},
})
if !runtime.tryMarkRunActive(run.ID) {
t.Fatal("expected active run mark to succeed")
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
if launchCount != 0 {
t.Fatalf("expected active run to be skipped, got %d launches", launchCount)
}
}
// TestProcessOnceSkippedLeaseActiveReleasesLeaseAndKeepsRunning guards the race
// where ProcessOnce's ClaimNextQueuedRun wins the mongo claim but tryMarkRunActive
// fails because another goroutine in this worker already owns the run.
//
// The old behavior reverted status to Queued while leaving currentNodeId set and
// leaving the just-claimed lease held — an inconsistent "queued+currentNodeId"
// state no other queued-writer emits (and the one that matched a phantom symptom
// we chased). It also blocked ProcessAvailable's claim filter for the lease TTL
// (~15min), orphaning the run. The fix releases the spurious lease and leaves
// status=Running (the active processor owns the run).
func TestProcessOnceSkippedLeaseActiveReleasesLeaseAndKeepsRunning(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "lease race", Description: "skip"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "workflow",
Description: "lease race",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
Nodes: []domain.WorkflowNode{{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{Label: "source"},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
launchCount := 0
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
launchCount++
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "job-" + input.RunID + "-" + input.NodeID, Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
},
})
// Pre-mark the run active so ProcessOnce's tryMarkRunActive fails, simulating
// the race where another goroutine already owns the in-memory activeRuns slot.
if !runtime.tryMarkRunActive(run.ID) {
t.Fatal("expected active run mark to succeed")
}
result, err := runtime.ProcessOnce(ctx)
if err != nil {
t.Fatalf("process once: %v", err)
}
if result == nil || result.Status != ProcessOnceStatusSkippedLeaseActive {
t.Fatalf("expected skipped_lease_active, got %#v", result)
}
if launchCount != 0 {
t.Fatalf("expected no launch (run already active), got %d", launchCount)
}
stored, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if stored.Status != domain.RunStatusRunning {
t.Fatalf("expected run to stay Running after skipped claim (active processor owns it), got %s", stored.Status)
}
if strings.TrimSpace(stored.ExecutionLeaseOwner) != "" {
t.Fatalf("expected spurious lease released so it does not block the next claim for the lease TTL, got owner=%q", stored.ExecutionLeaseOwner)
}
}
func TestExecuteNodeFallsBackToOperatorResourceDefaults(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
operator, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
Name: "OpenVLA 微调训练",
Type: "builtin",
Category: "AI",
Description: "gpu operator",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "model", Type: domain.PortTypeFile}},
Resources: &domain.OperatorResources{
CPURequest: "8",
CPULimit: "24",
MemoryRequest: "32Gi",
MemoryLimit: "128Gi",
GPULimit: "1",
SharedMemorySize: "12Gi",
},
})
if err != nil {
t.Fatalf("create operator: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "GPU 项目", Description: "resources"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "GPU 工作流",
Description: "operator resource defaults",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
Nodes: []domain.WorkflowNode{{
ID: "train",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "OpenVLA 微调训练",
OperatorID: operator.ID,
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "model", Type: domain.PortTypeFile}},
},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
var captured k8s.LaunchInput
launchCount := 0
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
launchCount++
captured = input
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o600); err != nil {
return nil, err
}
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "job-" + input.NodeID, Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
},
})
if _, _, err := runtime.executeNode(ctx, run, workflow.Nodes[0], nil, nil); err != nil {
t.Fatalf("execute node: %v", err)
}
if launchCount != 1 {
t.Fatalf("expected one launch, got %d", launchCount)
}
if captured.GPULimit != "1" || captured.Env["GPU_LIMIT"] != "1" {
t.Fatalf("expected operator GPU default to reach launcher/env, got gpu=%q env=%q", captured.GPULimit, captured.Env["GPU_LIMIT"])
}
if captured.CPURequest != "8" || captured.CPULimit != "24" || captured.MemoryRequest != "32Gi" || captured.MemoryLimit != "128Gi" {
t.Fatalf("expected operator resource defaults, got %#v", captured)
}
if captured.SharedMemorySize != "12Gi" {
t.Fatalf("expected operator shared memory default, got %q", captured.SharedMemorySize)
}
}
func TestResolveEffectiveNodeSharedMemoryPrecedence(t *testing.T) {
operatorResources := &domain.OperatorResources{SharedMemorySize: "8Gi"}
operatorDefault := resolveEffectiveNodeResources(domain.WorkflowNode{}, nil, operatorResources)
if operatorDefault.SharedMemorySize != "8Gi" {
t.Fatalf("operator shared memory default = %q, want 8Gi", operatorDefault.SharedMemorySize)
}
nodeOverride := resolveEffectiveNodeResources(domain.WorkflowNode{
Data: domain.WorkflowNodeData{SharedMemorySize: "16Gi"},
}, nil, operatorResources)
if nodeOverride.SharedMemorySize != "16Gi" {
t.Fatalf("node shared memory override = %q, want 16Gi", nodeOverride.SharedMemorySize)
}
platformFallback := resolveEffectiveNodeResources(domain.WorkflowNode{}, nil, nil)
if platformFallback.SharedMemorySize != "" {
t.Fatalf("empty shared memory should be left for launcher platform fallback, got %q", platformFallback.SharedMemorySize)
}
}
func TestProcessRunFailsInvalidWorkflowWithDuplicateInputEdges(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "非法工作流项目",
Description: "duplicate input edges",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "非法工作流",
Description: "duplicate input edges",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source-a",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "上游 A",
Category: "IO",
Description: "source a",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
},
},
{
ID: "source-b",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "上游 B",
Category: "IO",
Description: "source b",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
},
},
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "upload",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "result", Type: domain.PortTypeJSON}},
},
},
},
Edges: []domain.WorkflowEdge{
{ID: "edge-a", Source: "source-a", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
{ID: "edge-b", Source: "source-b", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
processed, err := runtime.processRun(ctx, run)
if err != nil {
t.Fatalf("process run: %v", err)
}
if processed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %s", processed.Status)
}
if !strings.Contains(processed.Error, "同一输入端口存在多条上游连线") {
t.Fatalf("expected duplicate input edge error, got %q", processed.Error)
}
if len(processed.NodeExecutions) != 0 {
t.Fatalf("expected no node executions for invalid workflow, got %#v", processed.NodeExecutions)
}
}
func TestProcessRunFailsInvalidIncrementalDirectoryDepth(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "非法目录深度项目",
Description: "invalid incremental depth",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "非法目录深度工作流",
Description: "invalid incremental depth",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
{Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"},
{Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "zero"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
processed := mustProcessOnceRun(t, runtime, ctx)
if processed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %s", processed.Status)
}
if !strings.Contains(processed.Error, "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") {
t.Fatalf("expected directory depth error, got %q", processed.Error)
}
execution := mustNodeExecution(t, processed, "source")
if execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed source execution, got %s", execution.Status)
}
if !strings.Contains(execution.Error, "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") {
t.Fatalf("expected node directory depth error, got %q", execution.Error)
}
}
func TestProcessRunFailsInvalidIncrementalUnit(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "非法增量单位项目",
Description: "invalid incremental unit",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "非法增量单位工作流",
Description: "invalid incremental unit",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
{Key: "INCREMENTAL_UNIT", Value: "ROW"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
processed := mustProcessOnceRun(t, runtime, ctx)
if processed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %s", processed.Status)
}
if !strings.Contains(processed.Error, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY") {
t.Fatalf("expected incremental unit error, got %q", processed.Error)
}
execution := mustNodeExecution(t, processed, "source")
if execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed source execution, got %s", execution.Status)
}
if !strings.Contains(execution.Error, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY") {
t.Fatalf("expected node incremental unit error, got %q", execution.Error)
}
}
func TestProcessRunFailsInvalidScanMode(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "非法扫描模式项目",
Description: "invalid scan mode",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "非法扫描模式工作流",
Description: "invalid scan mode",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "ROW"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
processed := mustProcessOnceRun(t, runtime, ctx)
if processed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %s", processed.Status)
}
if !strings.Contains(processed.Error, "SCAN_MODE must be FULL or INCREMENTAL") {
t.Fatalf("expected scan mode error, got %q", processed.Error)
}
execution := mustNodeExecution(t, processed, "source")
if execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed source execution, got %s", execution.Status)
}
if !strings.Contains(execution.Error, "SCAN_MODE must be FULL or INCREMENTAL") {
t.Fatalf("expected node scan mode error, got %q", execution.Error)
}
}
func TestProcessRunFailsInvalidDataSource(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "非法数据源项目",
Description: "invalid data source",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "非法数据源工作流",
Description: "invalid data source",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "ROW"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
processed := mustProcessOnceRun(t, runtime, ctx)
if processed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %s", processed.Status)
}
if !strings.Contains(processed.Error, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") {
t.Fatalf("expected data source error, got %q", processed.Error)
}
execution := mustNodeExecution(t, processed, "source")
if execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed source execution, got %s", execution.Status)
}
if !strings.Contains(execution.Error, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") {
t.Fatalf("expected node data source error, got %q", execution.Error)
}
}
func TestSourceStateFromCheckpointPrefersCheckpointFilePayload(t *testing.T) {
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
run := &domain.RunRecord{ID: "run-1"}
checkpointPath := filepath.Join(t.TempDir(), "source-checkpoint.json")
writeFile(t, checkpointPath, `{
"sourceFingerprint": "fp-1",
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"bucket": "datastorage",
"prefix": "algo/X-platform-test",
"pattern": "",
"scanMode": "INCREMENTAL",
"directoryDepth": 1,
"seenFingerprints": ["a", "b"],
"listedCount": 28,
"newCount": 0,
"scanStartedAt": "2026-04-12T15:45:42Z",
"scanCompletedAt": "2026-04-12T15:45:55Z"
}`)
execution := domain.NodeExecutionRecord{
NodeID: "scan-source",
Artifacts: []domain.ExecutionArtifact{
{
Name: "source-checkpoint.json",
Path: checkpointPath,
Data: map[string]any{
"sourceFingerprint": "fp-1",
"listedCount": 28.0,
"newCount": 28.0,
},
},
},
}
state, err := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0])
if err != nil {
t.Fatalf("source state from checkpoint: %v", err)
}
if state == nil {
t.Fatalf("expected source state")
}
if state.LastNewCount != 0 {
t.Fatalf("expected checkpoint file payload to win, got lastNewCount=%d", state.LastNewCount)
}
if state.LastListedCount != 28 {
t.Fatalf("expected lastListedCount=28, got %d", state.LastListedCount)
}
if state.LastRunID != run.ID {
t.Fatalf("expected lastRunID=%s, got %s", run.ID, state.LastRunID)
}
}
func TestSourceStateFromCheckpointRejectsUnreadableOrInvalidCheckpointFiles(t *testing.T) {
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
run := &domain.RunRecord{ID: "run-1"}
execution := domain.NodeExecutionRecord{NodeID: "scan-source"}
missingPath := filepath.Join(t.TempDir(), "missing-source-checkpoint.json")
_, err := sourceStateFromCheckpoint(workflow, run, execution, &domain.ExecutionArtifact{
Name: "source-checkpoint.json",
Path: missingPath,
Data: map[string]any{
"sourceFingerprint": "stale-inline",
"listedCount": 1,
"newCount": 1,
},
})
if err == nil || !strings.Contains(err.Error(), "read source checkpoint") {
t.Fatalf("expected unreadable checkpoint error, got %v", err)
}
invalidPath := filepath.Join(t.TempDir(), "source-checkpoint.json")
writeFile(t, invalidPath, `{`)
_, err = sourceStateFromCheckpoint(workflow, run, execution, &domain.ExecutionArtifact{
Name: "source-checkpoint.json",
Path: invalidPath,
Data: map[string]any{
"sourceFingerprint": "stale-inline",
"listedCount": 1,
"newCount": 1,
},
})
if err == nil || !strings.Contains(err.Error(), "parse source checkpoint") {
t.Fatalf("expected invalid checkpoint JSON error, got %v", err)
}
}
func TestSourceStateFromCheckpointRejectsLossyCheckpointCounts(t *testing.T) {
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
run := &domain.RunRecord{ID: "run-1"}
execution := domain.NodeExecutionRecord{
NodeID: "scan-source",
Artifacts: []domain.ExecutionArtifact{{
Name: "source-checkpoint.json",
Data: map[string]any{
"sourceFingerprint": "fp-lossy",
"directoryDepth": float64(1),
"listedCount": 28.5,
"newCount": 0.0,
},
}},
}
_, err := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0])
if err == nil || !strings.Contains(err.Error(), "listedCount must be an integer") {
t.Fatalf("expected lossy listedCount error, got %v", err)
}
}
func TestCommitSourceStatesPrefersPersistedRunSnapshot(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
persistedRun := domain.RunRecord{
ID: "run-1",
WorkflowID: workflow.ID,
NodeExecutions: []domain.NodeExecutionRecord{{
NodeID: "scan-source",
Artifacts: []domain.ExecutionArtifact{{
Name: "source-checkpoint.json",
Data: map[string]any{
"sourceFingerprint": "fp-1",
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"bucket": "datastorage",
"prefix": "algo/X-platform-test",
"scanMode": "INCREMENTAL",
"directoryDepth": 1.0,
"seenFingerprints": []any{"a", "b"},
"listedCount": 28.0,
"newCount": 0.0,
"scanStartedAt": "2026-04-12T15:45:42Z",
"scanCompletedAt": "2026-04-12T15:45:55Z",
},
}},
}},
}
if err := dataStore.UpsertRun(ctx, persistedRun); err != nil {
t.Fatalf("upsert persisted run: %v", err)
}
inMemoryRun := &domain.RunRecord{
ID: "run-1",
WorkflowID: workflow.ID,
NodeExecutions: []domain.NodeExecutionRecord{{
NodeID: "scan-source",
Artifacts: []domain.ExecutionArtifact{{
Name: "source-checkpoint.json",
Data: map[string]any{
"sourceFingerprint": "fp-1",
"directoryDepth": 1.0,
"listedCount": 28.0,
"newCount": 28.0,
},
}},
}},
}
if err := runtime.commitSourceStates(ctx, workflow, inMemoryRun); err != nil {
t.Fatalf("commit source states: %v", err)
}
state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-1")
if err != nil {
t.Fatalf("get source state: %v", err)
}
if state == nil {
t.Fatalf("expected committed source state")
}
if state.LastNewCount != 0 {
t.Fatalf("expected committed source state to use persisted run snapshot, got %d", state.LastNewCount)
}
}
func TestCommitSourceStatesPreservesSeenFingerprintsFromTypedSlice(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
workflow := &domain.WorkflowRecord{ID: "workflow-typed-slice"}
persistedRun := domain.RunRecord{
ID: "run-typed-slice",
WorkflowID: workflow.ID,
NodeExecutions: []domain.NodeExecutionRecord{{
NodeID: "scan-source",
Artifacts: []domain.ExecutionArtifact{{
Name: "source-checkpoint.json",
Data: map[string]any{
"sourceFingerprint": "fp-typed-slice",
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test/raw",
"bucket": "datastorage",
"prefix": "algo/X-platform-test/raw",
"scanMode": "INCREMENTAL",
"directoryDepth": 1.0,
"seenFingerprints": []string{"a", "b", "c"},
"listedCount": 4.0,
"newCount": 0.0,
},
}},
}},
}
if err := dataStore.UpsertRun(ctx, persistedRun); err != nil {
t.Fatalf("upsert persisted run: %v", err)
}
if err := runtime.commitSourceStates(ctx, workflow, &persistedRun); err != nil {
t.Fatalf("commit source states: %v", err)
}
state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-typed-slice")
if err != nil {
t.Fatalf("get source state: %v", err)
}
if state == nil {
t.Fatalf("expected committed source state")
}
if len(state.SeenFingerprints) != 3 {
t.Fatalf("expected 3 seen fingerprints, got %#v", state.SeenFingerprints)
}
if strings.Join(state.SeenFingerprints, ",") != "a,b,c" {
t.Fatalf("unexpected seen fingerprints %#v", state.SeenFingerprints)
}
}
func writeFile(t *testing.T, path string, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write %s: %v", path, err)
}
}
func createRosbagDB3(t *testing.T, path string) {
t.Helper()
command := exec.Command(
"python3",
"-c",
strings.Join([]string{
"import sqlite3, sys",
"conn = sqlite3.connect(sys.argv[1])",
"cur = conn.cursor()",
"cur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')",
"cur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')",
"cur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")",
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")",
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")",
"conn.commit()",
"conn.close()",
}, "; "),
path,
)
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("create rosbag db3 %s: %v: %s", path, err, string(output))
}
}
func TestChooseImagePrefersRuntimeOperatorRunnerForBuiltins(t *testing.T) {
node := domain.WorkflowNode{
Data: domain.WorkflowNodeData{
Image: "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:old-tag",
},
}
image := chooseImage(node, "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag")
if image != "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag" {
t.Fatalf("expected runtime image override for built-in operator-runner, got %q", image)
}
}
func TestChooseImagePrefersDigestPinnedOperatorImageID(t *testing.T) {
node := domain.WorkflowNode{
Data: domain.WorkflowNodeData{
Image: "ccr.example/dataflow-poc/droboticflow-operator-openvla-finetune:old",
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-openvla-finetune:fixed@sha256:" + strings.Repeat("a", 64),
},
}
image := chooseImage(node, "ccr.example/dataflow-poc/droboticflow-operator-runner:live")
if image != node.Data.OperatorImageID {
t.Fatalf("expected digest-pinned operator image id, got %q", image)
}
}
func TestChooseImagePrefersUntaggedDigestPinnedOperatorImageID(t *testing.T) {
node := domain.WorkflowNode{
Data: domain.WorkflowNodeData{
Image: "ccr.example/dataflow-poc/droboticflow-operator-act-policy-train:built-tag",
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-act-policy-train@sha256:" + strings.Repeat("b", 64),
},
}
image := chooseImage(node, "ccr.example/dataflow-poc/droboticflow-operator-runner:live")
if image != node.Data.OperatorImageID {
t.Fatalf("expected untagged digest-pinned operator image id, got %q", image)
}
}
func TestResolveBuiltinNodeRuntimeRefreshesStaleWorkflowImage(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
currentImage := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:builtin"
currentImageID := "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("c", 64)
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{
ID: "operator-builtin-bos-incremental-scan",
Name: "BOS 增量扫描",
Type: "builtin",
Category: "IO",
Description: "scan",
Image: currentImage,
ImageID: currentImageID,
ContainerCommand: []string{"python", "/app/backend/operator-runner/main.py"},
RuntimeContract: map[string]any{"handler": "builtin_source"},
}); err != nil {
t.Fatalf("upsert builtin operator: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
OperatorID: "operator-builtin-bos-incremental-scan",
Image: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan:deleted",
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-bos-incremental-scan@sha256:" + strings.Repeat("a", 64),
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: "bos://bucket/prefix"}},
}}
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
t.Fatalf("resolve builtin runtime: %v", err)
}
if resolved.Data.Image != currentImage || resolved.Data.OperatorImageID != currentImageID {
t.Fatalf("expected current builtin image, got image=%q imageId=%q", resolved.Data.Image, resolved.Data.OperatorImageID)
}
if strings.Join(resolved.Data.ContainerCommand, " ") != "python /app/backend/operator-runner/main.py" {
t.Fatalf("expected current container command, got %#v", resolved.Data.ContainerCommand)
}
if len(resolved.Data.Hyperparams) != 1 || resolved.Data.Hyperparams[0].Value != "bos://bucket/prefix" {
t.Fatalf("expected workflow parameters to be preserved, got %#v", resolved.Data.Hyperparams)
}
}
func TestResolveBuiltinNodeRuntimePreservesExplicitEnvironmentPin(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
currentImage := "registry.example/poc/operator:new"
currentImageID := "registry.example/poc/operator@sha256:" + strings.Repeat("a", 64)
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{ID: "operator-builtin-package", Name: "Package", Type: "builtin", Image: currentImage, ImageID: currentImageID, ContainerCommand: []string{"python", "/app/backend/operator-runner/main.py"}}); err != nil {
t.Fatal(err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
pinnedImage := "registry.example/fanyu/operator:release"
pinnedImageID := "registry.example/fanyu/operator@sha256:" + strings.Repeat("b", 64)
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorID: "operator-builtin-package", Image: pinnedImage, OperatorImageID: pinnedImageID, Hyperparams: []domain.HyperParam{{Key: "PIN_OPERATOR_IMAGE", Value: "true"}}}}
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
t.Fatal(err)
}
if resolved.Data.Image != pinnedImage || resolved.Data.OperatorImageID != pinnedImageID {
t.Fatalf("environment pin was overwritten: %#v", resolved.Data)
}
if strings.Join(resolved.Data.ContainerCommand, " ") != "python /app/backend/operator-runner/main.py" {
t.Fatalf("non-image runtime metadata was not refreshed: %#v", resolved.Data.ContainerCommand)
}
}
func TestResolveBuiltinNodeRuntimeLeavesCustomOperatorSnapshotUntouched(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
if err := dataStore.UpsertOperator(ctx, domain.OperatorRecord{
ID: "operator-custom", Name: "Custom", Type: "custom", Category: "IO", Description: "custom",
Image: "registry.example/custom:new", ImageID: "registry.example/custom@sha256:" + strings.Repeat("d", 64),
}); err != nil {
t.Fatalf("upsert custom operator: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
node := domain.WorkflowNode{Data: domain.WorkflowNodeData{
OperatorID: "operator-custom", Image: "registry.example/custom:pinned",
OperatorImageID: "registry.example/custom@sha256:" + strings.Repeat("e", 64),
}}
resolved, err := runtime.resolveBuiltinNodeRuntime(ctx, node)
if err != nil {
t.Fatalf("resolve custom runtime: %v", err)
}
if resolved.Data.Image != node.Data.Image || resolved.Data.OperatorImageID != node.Data.OperatorImageID {
t.Fatalf("custom operator snapshot changed: %#v", resolved.Data)
}
}
func createTestVideoMP4(t *testing.T, path string, pattern string, fps int, frames int, width int, height int) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
}
command := exec.Command(
"python3",
"-c",
strings.Join([]string{
"import cv2, numpy as np, sys",
"path, pattern, fps, frames, width, height = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]), int(sys.argv[6])",
"fourcc = cv2.VideoWriter_fourcc(*'mp4v')",
"writer = cv2.VideoWriter(path, fourcc, fps, (width, height))",
"assert writer.isOpened(), 'video writer did not open'",
"for index in range(frames):",
" frame = np.zeros((height, width, 3), dtype=np.uint8)",
" if pattern == 'sharp':",
" x = int((index * 3) % max(1, width - 18))",
" cv2.rectangle(frame, (x, 8), (x + 16, 24), (255, 255, 255), -1)",
" cv2.putText(frame, str(index % 10), (6, height - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1, cv2.LINE_AA)",
" elif pattern == 'blur':",
" cv2.rectangle(frame, (16, 16), (width - 16, height - 16), (220, 220, 220), -1)",
" frame = cv2.GaussianBlur(frame, (23, 23), 0)",
" else:",
" x = int((index * 5) % max(1, width - 20))",
" frame[:] = (40, 60, 90)",
" cv2.circle(frame, (x + 10, height // 2), 10, (80, 140, 190), -1)",
" writer.write(frame)",
"writer.release()",
"capture = cv2.VideoCapture(path)",
"assert capture.isOpened(), 'video capture did not open'",
"capture.release()",
"print(path)",
}, "\n"),
path,
pattern,
fmt.Sprintf("%d", fps),
fmt.Sprintf("%d", frames),
fmt.Sprintf("%d", width),
fmt.Sprintf("%d", height),
)
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("create test mp4 %s: %v: %s", path, err, string(output))
}
}
func createXPlatformDataset(t *testing.T) string {
t.Helper()
root := filepath.Join(t.TempDir(), "algo", "X-platform-test")
fps := 10
width := 96
height := 64
type videoSpec struct {
Name string
Pattern string
Frames int
Duration float64
}
type collectionSpec struct {
Name string
Type string
Place string
Date string
DeviceID string
HasIntrinsics bool
VideoDirectory string
Videos []videoSpec
}
collections := []collectionSpec{
{
Name: "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
Type: "家庭",
Place: "蓟门里小区",
Date: "20260319",
DeviceID: "C00000",
HasIntrinsics: true,
VideoDirectory: "DJI_001",
Videos: []videoSpec{
{Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 40, Duration: 4.0},
{Name: "DJI_0002_D.mp4", Pattern: "blur", Frames: 20, Duration: 2.0},
},
},
{
Name: "BJ_002_0002_OsmoNano_2026-03-20_10-30-00",
Type: "办公",
Place: "中关村",
Date: "20260320",
DeviceID: "C00001",
HasIntrinsics: false,
VideoDirectory: "DJI_001",
Videos: []videoSpec{
{Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 20, Duration: 2.0},
},
},
}
for _, collection := range collections {
collectionRoot := filepath.Join(root, collection.Name)
videoRoot := filepath.Join(collectionRoot, collection.VideoDirectory)
videoMetaRows := make([]map[string]any, 0, len(collection.Videos))
for index, video := range collection.Videos {
videoPath := filepath.Join(videoRoot, video.Name)
createTestVideoMP4(t, videoPath, video.Pattern, fps, video.Frames, width, height)
videoMetaRows = append(videoMetaRows, map[string]any{
"filename": video.Name,
"codec": "mp4v",
"resolution": fmt.Sprintf("%dx%d", width, height),
"frameRate": float64(fps),
"bitrate": 2000000 + index*100000,
"duration": video.Duration,
"startTime": 1773904218 + index*60,
})
}
metaJSON, err := json.Marshal(map[string]any{
"type": collection.Type,
"place": collection.Place,
"mission": "无",
"project_id": collection.Name,
"date": collection.Date,
"device_id": collection.DeviceID,
})
if err != nil {
t.Fatalf("marshal meta json: %v", err)
}
writeFile(t, filepath.Join(collectionRoot, "meta.json"), string(metaJSON))
videoMetaJSON, err := json.Marshal(videoMetaRows)
if err != nil {
t.Fatalf("marshal video_meta json: %v", err)
}
writeFile(t, filepath.Join(collectionRoot, "video_meta.json"), string(videoMetaJSON))
writeFile(t, filepath.Join(videoRoot, "DJI_meta.json"), `{"ok":true}`)
if collection.HasIntrinsics {
writeFile(t, filepath.Join(collectionRoot, "intrinsics.json"), `{"fx": 1000, "fy": 1000}`)
}
}
return root
}
func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
if run != nil && run.Status == expected {
return run
}
time.Sleep(100 * time.Millisecond)
}
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
t.Fatalf("timed out waiting for run %s to become %s, got %#v", runID, expected, run)
return nil
}
func waitForRunningNodeCount(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected int, timeout time.Duration) *domain.RunRecord {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
if run == nil {
time.Sleep(100 * time.Millisecond)
continue
}
running := 0
for _, execution := range run.NodeExecutions {
if execution.Status == domain.NodeExecutionStatusRunning {
running++
}
}
if running == expected {
return run
}
time.Sleep(100 * time.Millisecond)
}
run, err := dataStore.GetRun(ctx, runID)
if err != nil {
t.Fatalf("get run %s: %v", runID, err)
}
t.Fatalf("timed out waiting for run %s to have %d running nodes, got %#v", runID, expected, run)
return nil
}
func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Worker 项目", Description: "worker"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "批处理工作流",
Description: "complete path",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "OSS 数据源",
Category: "IO",
Description: "source",
Inputs: []domain.PortDefinition{},
Outputs: []domain.PortDefinition{
{ID: "out-1", Name: "file", Type: domain.PortTypeFile},
},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "INLINE"},
{Key: "SOURCE_TEXT", Value: "worker-runtime-test"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "sink",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "sink",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{},
},
Position: domain.Position{X: 100, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "e1", Source: "source", SourceHandle: "out-1", Target: "sink", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
for _, item := range completed.NodeExecutions {
if item.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected completed node execution, got %s", item.Status)
}
}
if len(completed.Artifacts) == 0 {
t.Fatalf("expected artifacts")
}
if completed.Artifacts[0].Path == "" {
t.Fatalf("expected real artifact path")
}
if len(completed.ResourceSeries) == 0 {
t.Fatalf("expected resource samples from operator runner")
}
}
func TestRuntimeStopsGracefullyWithoutDispatchingNewNodes(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "停止项目", Description: "graceful"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "停止工作流",
Description: "graceful stop path",
Type: domain.WorkflowTypeBatch,
MaxConcurrentRuns: 1,
NodeTimeoutSeconds: 30,
RunTimeoutSeconds: 60,
MaxRetryAttempts: 0,
RetryBackoffSeconds: 0,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "source",
Outputs: []domain.PortDefinition{
{ID: "payload", Name: "payload", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "COMMAND", Value: `sleep 1; printf '{"value":1}' > payload.json`},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "sink",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "sink",
Inputs: []domain.PortDefinition{
{ID: "input", Name: "input", Type: domain.PortTypeJSON},
},
},
Position: domain.Position{X: 120, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "e1", Source: "source", SourceHandle: "payload", Target: "sink", TargetHandle: "input"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
done := make(chan error, 1)
go func() {
_, processErr := runtime.ProcessOnce(ctx)
done <- processErr
}()
waitForRunningNodeCount(t, ctx, dataStore, run.ID, 1, 5*time.Second)
stored, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get running run: %v", err)
}
stored.Status = domain.RunStatus("stopping")
stored.StopMode = domain.RunStopModeGraceful
stored.StopRequestedAt = nowISO()
stored.UpdatedAt = nowISO()
if err := dataStore.UpsertRun(ctx, *stored); err != nil {
t.Fatalf("request graceful stop: %v", err)
}
select {
case err := <-done:
if err != nil {
t.Fatalf("process run: %v", err)
}
case <-time.After(10 * time.Second):
t.Fatalf("timed out waiting for graceful stop")
}
finalRun, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get final run: %v", err)
}
if finalRun == nil || finalRun.Status != domain.RunStatusCancelled {
t.Fatalf("expected cancelled run after graceful stop, got %#v", finalRun)
}
if sink := mustNodeExecution(t, finalRun, "source"); sink.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected source node to complete before stop, got %s", sink.Status)
}
for _, item := range finalRun.NodeExecutions {
if item.NodeID == "sink" {
t.Fatalf("expected sink node not to start after graceful stop, got %#v", item)
}
}
}
func TestRuntimePassesContainerCommandAndArgsToLauncher(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "镜像项目", Description: "command args"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "自定义容器入口",
Description: "verify command args",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "custom",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "ACT 策略训练",
Description: "custom entrypoint",
Image: "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime",
ContainerCommand: []string{"/bin/bash", "-lc"},
ContainerArgs: []string{"python /workspace/operators/act.py"},
RuntimeContract: map[string]any{"handler": "builtin_act_policy_train"},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
var captured k8s.LaunchInput
runtime := &Runtime{
store: dataStore,
operatorRunnerImage: "operator-runner:latest",
workspaceRoot: t.TempDir(),
launcher: testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
captured = input
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
return nil, err
}
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "custom", Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
},
},
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
if captured.Image != "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime" {
t.Fatalf("image = %q", captured.Image)
}
if strings.Join(captured.Command, " ") != "/bin/bash -lc" {
t.Fatalf("command = %#v", captured.Command)
}
if len(captured.Args) != 1 || captured.Args[0] != "python /workspace/operators/act.py" {
t.Fatalf("args = %#v", captured.Args)
}
if captured.Env["DROBOTICFLOW_OPERATOR_HANDLER"] != "act_policy_train" {
t.Fatalf("handler env = %q", captured.Env["DROBOTICFLOW_OPERATOR_HANDLER"])
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
}
func TestRuntimePassesLightweightOperatorMountToLauncher(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "轻量项目", Description: "lightweight"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "轻量算子工作流",
Description: "verify lightweight code mount",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "light",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "轻量算子",
Description: "code from workspace",
Image: "python:3.11-slim",
ContainerCommand: []string{"/bin/sh", "-lc"},
ContainerArgs: []string{"python operator.py"},
RuntimeContract: map[string]any{
"lightweightOperator": map[string]any{
"enabled": true,
"codeSubPath": "lightweight-operators/demo",
"mountPath": "/app/operator",
"mounts": []any{map[string]any{"name": "data", "pvcName": "dataset-pvc", "subPath": "robotwin", "mountPath": "/data", "readOnly": true}},
},
},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("create run: %v", err)
}
var captured k8s.LaunchInput
runtime := &Runtime{
store: dataStore,
operatorRunnerImage: "operator-runner:latest",
workspaceRoot: t.TempDir(),
launcher: testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
captured = input
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
return nil, err
}
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "light", Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
},
},
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
if captured.CodeSubPath != "lightweight-operators/demo" || captured.CodeMountPath != "/app/operator" {
t.Fatalf("unexpected lightweight mount: subPath=%q mountPath=%q", captured.CodeSubPath, captured.CodeMountPath)
}
if captured.Env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] != "/app/operator" {
t.Fatalf("LIGHTWEIGHT_OPERATOR_CODE_DIR = %q", captured.Env["LIGHTWEIGHT_OPERATOR_CODE_DIR"])
}
if len(captured.PVCMounts) != 1 || captured.PVCMounts[0].PVCName != "dataset-pvc" || captured.PVCMounts[0].MountPath != "/data" || !captured.PVCMounts[0].ReadOnly {
t.Fatalf("unexpected additional PVC mounts: %#v", captured.PVCMounts)
}
}
func TestRuntimeRejectsLightweightOperatorWithoutExplicitContainerArgs(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "轻量项目", Description: "lightweight"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "轻量算子工作流",
Description: "missing explicit args",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "light",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "轻量算子",
Description: "code from workspace",
Image: "python:3.11-slim",
ContainerCommand: []string{"python", "/app/operator/operator.py"},
RuntimeContract: map[string]any{
"lightweightOperator": map[string]any{
"enabled": true,
"codeSubPath": "lightweight-operators/demo",
"mountPath": "/app/operator",
},
},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("create run: %v", err)
}
launchCalled := false
runtime := &Runtime{
store: dataStore,
operatorRunnerImage: "operator-runner:latest",
workspaceRoot: t.TempDir(),
launcher: testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
launchCalled = true
return nil, fmt.Errorf("launch should not be called")
},
},
}
processed := mustProcessOnceRun(t, runtime, ctx)
if launchCalled {
t.Fatalf("expected lightweight preflight to fail before launch")
}
if processed.Status != domain.RunStatusFailed || !strings.Contains(processed.Error, "containerArgs must be explicitly configured") {
t.Fatalf("expected explicit args preflight failure, got status=%s error=%q", processed.Status, processed.Error)
}
}
func TestRuntimePausesForAuditAndResumes(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Audit 项目", Description: "audit"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "审核工作流",
Description: "audit path",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "review",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "人工审核 (Human Review)",
Category: "HUMAN",
Description: "review",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{
{ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "sink",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{},
},
Position: domain.Position{X: 100, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "e1", Source: "review", SourceHandle: "approved", Target: "upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
paused, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get paused run: %v", err)
}
if paused == nil || paused.Status != domain.RunStatusWaitingForAudit {
t.Fatalf("expected waiting_for_audit run, got %#v", paused)
}
tasks, err := dataStore.ListAuditTasks(ctx, "all")
if err != nil {
t.Fatalf("list audit tasks: %v", err)
}
var task *domain.AuditTaskRecord
for _, item := range tasks {
if item.RunID == run.ID && item.Status == domain.AuditStatusPending {
copied := item
task = &copied
break
}
}
if task == nil {
t.Fatalf("expected pending audit task")
}
if task.OrganizationID != run.OrganizationID {
t.Fatalf("audit task organization = %q, want run organization %q", task.OrganizationID, run.OrganizationID)
}
batches, err := dataStore.ListHumanBatches(ctx, run.OrganizationID, domain.HumanBatchQuery{RunID: run.ID, PluginID: "human-loop-core"})
if err != nil || len(batches) != 1 {
t.Fatalf("generic human batches=%#v err=%v", batches, err)
}
humanItems, err := dataStore.ListHumanItems(ctx, run.OrganizationID, domain.HumanItemQuery{BatchID: batches[0].ID})
if err != nil || len(humanItems) != 1 || humanItems[0].LegacyAuditTaskID != task.ID {
t.Fatalf("generic human items=%#v err=%v", humanItems, err)
}
task.Status = domain.AuditStatusApproved
task.UpdatedAt = nowISO()
if err := dataStore.UpsertAuditTask(ctx, *task); err != nil {
t.Fatalf("approve audit task: %v", err)
}
paused.Status = domain.RunStatusQueued
paused.UpdatedAt = nowISO()
if err := dataStore.UpsertRun(ctx, *paused); err != nil {
t.Fatalf("queue paused run: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("resume process once: %v", err)
}
resumed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get resumed run: %v", err)
}
if resumed == nil || resumed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed resumed run, got %#v", resumed)
}
}
func TestRuntimeRoutesArtifactsAcrossSequentialHumanReviews(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "双重审核项目", Description: "sequential audit"})
if err != nil {
t.Fatalf("create project: %v", err)
}
reviewNode := func(id string, label string) domain.WorkflowNode {
return domain.WorkflowNode{
ID: id,
Type: "operator",
Data: domain.WorkflowNodeData{
Label: label,
Category: "HUMAN",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "candidate", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{
{ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON},
{ID: "rejected", Name: "审核拒绝", Type: domain.PortTypeJSON},
},
},
}
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "双重人工审核工作流",
Description: "preserve artifacts through both gates",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: `{"items":[{"name":"clip.mp4"}],"count":1}`}},
},
},
reviewNode("quality-review", "切分与语义质检"),
reviewNode("acceptance-review", "最终验收"),
{
ID: "quality-rejected-upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "质检拒绝上传(不应执行)",
Category: "IO",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "rejected", Type: domain.PortTypeJSON}},
},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-quality", Source: "source", SourceHandle: "out-1", Target: "quality-review", TargetHandle: "in-1"},
{ID: "quality-acceptance", Source: "quality-review", SourceHandle: "approved", Target: "acceptance-review", TargetHandle: "in-1"},
{ID: "quality-rejected", Source: "quality-review", SourceHandle: "rejected", Target: "quality-rejected-upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process to first audit: %v", err)
}
paused, err := dataStore.GetRun(ctx, run.ID)
if err != nil || paused == nil {
t.Fatalf("get first paused run: run=%#v err=%v", paused, err)
}
firstTasks, err := dataStore.ListAuditTasks(ctx, string(domain.AuditStatusPending))
if err != nil {
t.Fatalf("list first audit tasks: %v", err)
}
if len(firstTasks) != 1 || firstTasks[0].NodeID != "quality-review" {
t.Fatalf("expected quality review task, got %#v", firstTasks)
}
firstTasks[0].Status = domain.AuditStatusApproved
firstTasks[0].UpdatedAt = nowISO()
if err := dataStore.UpsertAuditTask(ctx, firstTasks[0]); err != nil {
t.Fatalf("approve quality review: %v", err)
}
paused.Status = domain.RunStatusQueued
paused.UpdatedAt = nowISO()
if err := dataStore.UpsertRun(ctx, *paused); err != nil {
t.Fatalf("queue after quality review: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process to final audit: %v", err)
}
finalRun, err := dataStore.GetRun(ctx, run.ID)
if err != nil || finalRun == nil {
t.Fatalf("get final audit run: run=%#v err=%v", finalRun, err)
}
qualityExecution := mustNodeExecution(t, finalRun, "quality-review")
if len(qualityExecution.Artifacts) != 1 || qualityExecution.Artifacts[0].PortID != "approved" {
t.Fatalf("expected approved routed artifact, got %#v", qualityExecution.Artifacts)
}
acceptanceExecution := mustNodeExecution(t, finalRun, "acceptance-review")
if len(acceptanceExecution.InputArtifacts) != 1 {
t.Fatalf("expected final review to receive one artifact, got %#v", acceptanceExecution.InputArtifacts)
}
rejectedExecution := mustNodeExecution(t, finalRun, "quality-rejected-upload")
if rejectedExecution.Status != domain.NodeExecutionStatusCompleted || len(rejectedExecution.Artifacts) != 0 || len(rejectedExecution.InputArtifacts) != 0 {
t.Fatalf("expected unselected rejection branch to be skipped without artifacts, got %#v", rejectedExecution)
}
if len(rejectedExecution.Logs) != 1 || !strings.Contains(rejectedExecution.Logs[0].Message, "Skipped by audit decision: approved") {
t.Fatalf("expected explicit skipped-branch log, got %#v", rejectedExecution.Logs)
}
finalTasks, err := dataStore.ListAuditTasks(ctx, string(domain.AuditStatusPending))
if err != nil {
t.Fatalf("list final audit tasks: %v", err)
}
if len(finalTasks) != 1 || finalTasks[0].NodeID != "acceptance-review" {
t.Fatalf("expected final acceptance task, got %#v", finalTasks)
}
if len(finalTasks[0].Data.DetectedObjects) != 1 || finalTasks[0].Data.DetectedObjects[0] != "1 upstream artifact(s)" {
t.Fatalf("expected final audit artifact summary, got %#v", finalTasks[0].Data.DetectedObjects)
}
}
func TestRuntimeAnnotationGateCreatesTasksAndReturnsAcceptedRevisionArtifact(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: domain.DefaultOrganizationID, Name: "Native Annotation"})
if err != nil {
t.Fatalf("create project: %v", err)
}
template := domain.AnnotationTemplateRecord{
// Organization-level published templates are reusable by tenant projects.
ID: "template-1", OrganizationID: project.OrganizationID, ProjectID: "",
Name: "动作模板", Version: 1, Status: domain.AnnotationTemplatePublished,
Labels: []domain.SemanticLabelDefinition{{ID: "steady", Name: "平稳活动"}, {ID: "fast", Name: "快速移动"}},
HumanStages: domain.DefaultFanyuV58HumanStages(),
}
supplier := domain.SupplierRecord{ID: "supplier-a", OrganizationID: project.OrganizationID, Name: "供应商 A", Status: "active"}
if err := dataStore.UpsertAnnotationTemplate(ctx, template); err != nil {
t.Fatalf("upsert template: %v", err)
}
if err := dataStore.UpsertSupplier(ctx, supplier); err != nil {
t.Fatalf("upsert supplier: %v", err)
}
for index, role := range []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator, domain.AnnotationRoleQualityReviewer} {
if err := dataStore.UpsertProjectMember(ctx, domain.ProjectMemberRecord{
ID: fmt.Sprintf("member-%d", index), OrganizationID: project.OrganizationID, ProjectID: project.ID,
UserID: fmt.Sprintf("user-%d", index), SupplierID: supplier.ID, Roles: []domain.AnnotationAssignmentRole{role},
}); err != nil {
t.Fatalf("upsert project member: %v", err)
}
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
OrganizationID: project.OrganizationID,
ProjectID: project.ID,
Name: "Native Annotation",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "source", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "clips", Type: domain.PortTypeJSON}}}},
{ID: "annotation-gate", Type: "operator", Data: domain.WorkflowNodeData{
Label: "逐段语义标注", Category: "HUMAN",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "clips", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "approved", Name: "approved", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "REVIEW_MODE", Value: "annotation"},
{Key: "ANNOTATION_TEMPLATE_ID", Value: template.ID},
{Key: "ANNOTATION_SUPPLIER_ID", Value: supplier.ID},
},
}},
},
Edges: []domain.WorkflowEdge{{ID: "source-gate", Source: "source", SourceHandle: "out-1", Target: "annotation-gate", TargetHandle: "in-1"}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
root := t.TempDir()
videoPath := filepath.Join(root, "clip001.mp4")
if err := os.WriteFile(videoPath, []byte("real-clip-placeholder"), 0o644); err != nil {
t.Fatalf("write video fixture: %v", err)
}
clipsPath := filepath.Join(root, "clips-manifest.json")
clipsPayload := map[string]any{
"clipCount": 1,
"items": []any{map[string]any{
"clipId": "clip001", "clipName": "clip001.mp4", "clipPath": videoPath,
"startSecond": 0.0, "endSecond": 10.0, "duration": 10.0,
}},
}
clipsBytes, _ := json.Marshal(clipsPayload)
if err := os.WriteFile(clipsPath, clipsBytes, 0o644); err != nil {
t.Fatalf("write clips manifest: %v", err)
}
prelabelPath := filepath.Join(root, "clip001_annotation.json")
prelabelPayload := map[string]any{
"clipId": "clip001",
"actions": []any{map[string]any{"label": "平稳活动", "start": 0.0, "end": 10.0, "confidence": 0.8}},
}
prelabelBytes, _ := json.Marshal(prelabelPayload)
if err := os.WriteFile(prelabelPath, prelabelBytes, 0o644); err != nil {
t.Fatalf("write prelabel: %v", err)
}
run.Status = domain.RunStatusQueued
run.CurrentNodeID = ""
run.NodeExecutions = []domain.NodeExecutionRecord{{
NodeID: "source", NodeLabel: "source", Attempt: 1, Status: domain.NodeExecutionStatusCompleted,
StartedAt: nowISO(), EndedAt: nowISO(), Logs: []domain.LogEntryRecord{}, ResourceSeries: []domain.ResourceSample{},
Artifacts: []domain.ExecutionArtifact{
{ID: "artifact-video", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "clip001.mp4", Type: domain.PortTypeVideo, Path: videoPath, Data: map[string]any{"preview": map[string]any{"duration": 10.0}}},
{ID: "artifact-headpose-render", RunID: run.ID, NodeID: "source", PortID: "headpose", Name: "headpose-annotated.mp4", Type: domain.PortTypeVideo, Path: videoPath, Data: map[string]any{"preview": map[string]any{"duration": 10.0}}},
{ID: "artifact-clips", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "clips-manifest.json", Type: domain.PortTypeJSON, Path: clipsPath, Data: clipsPayload},
{ID: "artifact-prelabel", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "clip001_annotation.json", Type: domain.PortTypeJSON, Path: prelabelPath, Data: prelabelPayload},
},
}}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert seeded run: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) {
return nil, fmt.Errorf("annotation gate should not launch a Kubernetes job")
}})
runtime.workspaceRoot = root
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process to annotation gate: %v", err)
}
waiting, err := dataStore.GetRun(ctx, run.ID)
if err != nil || waiting == nil || waiting.Status != domain.RunStatusWaitingForHuman {
t.Fatalf("annotation gate did not pause run: run=%#v err=%v", waiting, err)
}
tasks, err := dataStore.ListAnnotationTasks(ctx, project.OrganizationID, project.ID)
if err != nil || len(tasks) != 1 {
t.Fatalf("annotation tasks = %#v err=%v", tasks, err)
}
if tasks[0].RunID != run.ID || tasks[0].NodeID != "annotation-gate" || tasks[0].ArtifactID != "artifact-video" || tasks[0].DataTaskID != run.TaskID {
t.Fatalf("annotation task linkage = %#v", tasks[0])
}
batches, err := dataStore.ListHumanBatches(ctx, project.OrganizationID, domain.HumanBatchQuery{RunID: run.ID})
if err != nil || len(batches) != 1 || batches[0].ReleasePolicy != domain.HumanReleaseAcceptedPartition {
t.Fatalf("human batches = %#v err=%v", batches, err)
}
humanItems, err := dataStore.ListHumanItems(ctx, project.OrganizationID, domain.HumanItemQuery{BatchID: batches[0].ID})
if err != nil || len(humanItems) != 1 || humanItems[0].LegacyTaskID != tasks[0].ID || humanItems[0].CurrentStageID != domain.FanyuV58StageAnnotationCombined {
t.Fatalf("human items = %#v err=%v", humanItems, err)
}
revisions, err := dataStore.ListAnnotationRevisions(ctx, project.OrganizationID, tasks[0].ID)
if err != nil || len(revisions) != 1 || revisions[0].Revision != 0 || len(revisions[0].Segments) != 1 {
t.Fatalf("prelabel revision = %#v err=%v", revisions, err)
}
acceptedSegments := []domain.AnnotationSegment{{ID: "segment-1", LabelID: "fast", Label: "快速移动", Start: 0, End: 10, Confidence: 1, Source: "manual"}}
if err := dataStore.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{
ID: "revision-accepted", OrganizationID: project.OrganizationID, TaskID: tasks[0].ID,
Revision: 1, TemplateVersion: 1, Segments: acceptedSegments, AuthorUserID: "annotator", CountsAsOutput: true, CreatedAt: nowISO(),
}); err != nil {
t.Fatalf("upsert accepted revision: %v", err)
}
tasks[0].CurrentRevision = 1
tasks[0].Status = domain.AnnotationTaskStatusAccepted
if err := dataStore.UpsertAnnotationTask(ctx, tasks[0]); err != nil {
t.Fatalf("accept annotation task: %v", err)
}
waiting.Status = domain.RunStatusQueued
if err := dataStore.UpsertRun(ctx, *waiting); err != nil {
t.Fatalf("queue accepted annotation run: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("resume accepted annotation run: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil || completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("completed annotation run = %#v err=%v", completed, err)
}
gateExecution := mustNodeExecution(t, completed, "annotation-gate")
manifest := mustArtifact(t, gateExecution, "annotations-manifest.json")
if manifest.Metadata == nil || manifest.Metadata.SHA256 == "" || manifest.PortID != "approved" {
t.Fatalf("accepted annotation artifact = %#v", manifest)
}
manifestItems := anySlice(manifest.Data["items"])
if len(manifestItems) != 1 {
t.Fatalf("accepted manifest items = %#v", manifest.Data["items"])
}
}
func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Route 项目", Description: "route"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "条件分流工作流",
Description: "route by handle",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "source",
Outputs: []domain.PortDefinition{
{ID: "out-manifest", Name: "manifest", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "COMMAND", Value: `printf '{"items":[{"kind":"keep","value":1},{"kind":"drop","value":2}]}' > routed.json`},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "route",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "条件分流",
Category: "LOGIC",
Description: "route",
Inputs: []domain.PortDefinition{
{ID: "in-1", Name: "input", Type: domain.PortTypeJSON},
},
Outputs: []domain.PortDefinition{
{ID: "matched", Name: "matched", Type: domain.PortTypeJSON},
{ID: "unmatched", Name: "unmatched", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "FIELD_PATH", Value: "kind"},
{Key: "OPERATOR", Value: "eq"},
{Key: "EXPECTED_JSON", Value: `"keep"`},
},
},
Position: domain.Position{X: 200, Y: 0},
},
{
ID: "keep-upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "keep sink",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 420, Y: -80},
},
{
ID: "drop-upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
Category: "IO",
Description: "drop sink",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 420, Y: 80},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-route", Source: "source", SourceHandle: "out-manifest", Target: "route", TargetHandle: "in-1"},
{ID: "route-keep", Source: "route", SourceHandle: "matched", Target: "keep-upload", TargetHandle: "in-1"},
{ID: "route-drop", Source: "route", SourceHandle: "unmatched", Target: "drop-upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
routeExecution := mustNodeExecution(t, completed, "route")
matched := mustArtifact(t, routeExecution, "matched.json")
unmatched := mustArtifact(t, routeExecution, "unmatched.json")
if matched.PortID != "matched" || unmatched.PortID != "unmatched" {
t.Fatalf("expected output port IDs on split artifacts, got matched=%q unmatched=%q", matched.PortID, unmatched.PortID)
}
keepUpload := mustNodeExecution(t, completed, "keep-upload")
dropUpload := mustNodeExecution(t, completed, "drop-upload")
keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json")
dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json")
keepSummaryPayload := mustArtifactPayload(t, keepSummary)
dropSummaryPayload := mustArtifactPayload(t, dropSummary)
keepItems, _ := keepSummaryPayload["items"].([]any)
dropItems, _ := dropSummaryPayload["items"].([]any)
if len(keepItems) != 1 || len(dropItems) != 1 {
t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems))
}
keepPayload, _ := keepItems[0].(map[string]any)
dropPayload, _ := dropItems[0].(map[string]any)
keepRows, _ := keepPayload["items"].([]any)
dropRows, _ := dropPayload["items"].([]any)
if len(keepRows) != 1 || len(dropRows) != 1 {
t.Fatalf("expected one row in each routed payload, got keep=%d drop=%d", len(keepRows), len(dropRows))
}
if keepRow, _ := keepRows[0].(map[string]any); keepRow != nil {
if keepRow["kind"] != "keep" {
t.Fatalf("unexpected keep row: %#v", keepRow)
}
}
if dropRow, _ := dropRows[0].(map[string]any); dropRow != nil {
if dropRow["kind"] != "drop" {
t.Fatalf("unexpected drop row: %#v", dropRow)
}
}
}
func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) {
t.Run("command previews text and csv outputs", func(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Preview 项目", Description: "preview"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "文件预览工作流",
Description: "preview files",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{
{Key: "COMMAND", Value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
commandExecution := mustNodeExecution(t, completed, "command")
report := mustArtifact(t, commandExecution, "report.csv")
notes := mustArtifact(t, commandExecution, "notes.txt")
reportPreview, _ := report.Data["preview"].(map[string]any)
notesPreview, _ := notes.Data["preview"].(map[string]any)
if reportPreview["kind"] != "table" {
t.Fatalf("expected table preview for csv artifact, got %#v", reportPreview)
}
if notesPreview["kind"] != "text" {
t.Fatalf("expected text preview for text artifact, got %#v", notesPreview)
}
})
t.Run("lerobot dataset summary", func(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
datasetRoot := t.TempDir()
writeFile(t, filepath.Join(datasetRoot, "meta", "info.json"), `{"robot_type":"aloha","fps":30}`)
writeFile(t, filepath.Join(datasetRoot, "meta", "stats.json"), `{"episodes":2}`)
writeFile(t, filepath.Join(datasetRoot, "meta", "tasks.jsonl"), "{\"task\":\"pick\"}\n{\"task\":\"place\"}\n")
writeFile(t, filepath.Join(datasetRoot, "data", "episode_000.parquet"), "PAR1")
writeFile(t, filepath.Join(datasetRoot, "videos", "episode_000.mp4"), "video")
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "LeRobot 项目", Description: "lerobot"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "LeRobot 概览",
Description: "lerobot summary",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: datasetRoot},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "summary",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "LeRobot 数据集概览",
Category: "ROBOTICS",
Description: "lerobot",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 240, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json")
summaryPayload := mustArtifactPayload(t, summary)
preview, _ := summaryPayload["preview"].(map[string]any)
if preview["format"] != "lerobot" {
t.Fatalf("expected lerobot preview, got %#v", preview)
}
if summaryPayload["datasetType"] != "lerobot" {
t.Fatalf("expected datasetType lerobot, got %#v", summaryPayload)
}
})
t.Run("rosbag sqlite summary", func(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
datasetRoot := t.TempDir()
dbPath := filepath.Join(datasetRoot, "bag_0.db3")
createRosbagDB3(t, dbPath)
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "ROS 项目", Description: "rosbag"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "ROSBag 概览",
Description: "rosbag summary",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: dbPath},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "summary",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "ROSBag 数据概览",
Category: "ROBOTICS",
Description: "rosbag",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}},
},
Position: domain.Position{X: 240, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json")
summaryPayload := mustArtifactPayload(t, summary)
preview, _ := summaryPayload["preview"].(map[string]any)
if preview["format"] != "rosbag2" {
t.Fatalf("expected rosbag2 preview, got %#v", preview)
}
topics, _ := preview["topics"].([]any)
if len(topics) != 1 {
t.Fatalf("expected one topic summary, got %#v", topics)
}
})
}
func TestRuntimeProcessesXPlatformPipeline(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
datasetRoot := createXPlatformDataset(t)
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "XPlatform 项目", Description: "x-platform"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "XPlatform 处理流水线",
Description: "scan -> validate -> quality -> filter -> slice -> annotate -> package -> sample",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: datasetRoot},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "index",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "X 平台数据集索引",
Category: "DATASET",
Description: "index dataset",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "dataset", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 220, Y: 0},
},
{
ID: "validate",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "采集目录校验",
Category: "DATASET",
Description: "validate dataset",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "validated", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 440, Y: 0},
},
{
ID: "quality",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "视频质检",
Category: "VIDEO",
Description: "quality",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "quality", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "MIN_CLARITY", Value: "0.8"},
{Key: "MAX_SHAKE", Value: "0.1"},
{Key: "MAX_HAND_OCCLUSION", Value: "0.1"},
},
},
Position: domain.Position{X: 660, Y: 0},
},
{
ID: "filter",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "数据过滤",
Category: "VIDEO",
Description: "filter dataset",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "filtered", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATE_START", Value: "2026-02-01"},
{Key: "DATE_END", Value: "2026-04-05"},
{Key: "PLACE", Value: "蓟门里小区"},
{Key: "TYPE", Value: "家庭"},
{Key: "MIN_CLARITY", Value: "0.8"},
{Key: "MAX_SHAKE", Value: "0.1"},
{Key: "MAX_HAND_OCCLUSION", Value: "0.1"},
},
},
Position: domain.Position{X: 880, Y: 0},
},
{
ID: "slice",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "视频切片",
Category: "VIDEO",
Description: "slice video",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "clips", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "SEGMENT_DURATION_SECONDS", Value: "2"}},
},
Position: domain.Position{X: 1100, Y: 0},
},
{
ID: "annotate",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "视频标注",
Category: "VIDEO",
Description: "annotate clips",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "FRAME_INTERVAL_SECONDS", Value: "0.5"}},
},
Position: domain.Position{X: 1320, Y: 0},
},
{
ID: "package",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "成品打包",
Category: "VIDEO",
Description: "package results",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "packages", Type: domain.PortTypeJSON}},
},
Position: domain.Position{X: 1540, Y: 0},
},
{
ID: "sample",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "抽检采样",
Category: "QA",
Description: "sample packages",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "RULES_JSON", Value: `[{"id":"clarity-check","field":"quality.clarity","operator":"lt","value":1,"samplePercent":100},{"id":"confidence-check","field":"annotation.minConfidence","operator":"lt","value":1,"samplePercent":100}]`},
},
},
Position: domain.Position{X: 1760, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-index", Source: "source", SourceHandle: "out-1", Target: "index", TargetHandle: "in-1"},
{ID: "index-validate", Source: "index", SourceHandle: "out-1", Target: "validate", TargetHandle: "in-1"},
{ID: "validate-quality", Source: "validate", SourceHandle: "out-1", Target: "quality", TargetHandle: "in-1"},
{ID: "quality-filter", Source: "quality", SourceHandle: "out-1", Target: "filter", TargetHandle: "in-1"},
{ID: "filter-slice", Source: "filter", SourceHandle: "out-1", Target: "slice", TargetHandle: "in-1"},
{ID: "slice-annotate", Source: "slice", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"},
{ID: "annotate-package", Source: "annotate", SourceHandle: "out-1", Target: "package", TargetHandle: "in-1"},
{ID: "package-sample", Source: "package", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected completed run, got %#v", completed)
}
indexArtifact := mustArtifact(t, mustNodeExecution(t, completed, "index"), "dataset-index.json")
indexPayload := mustArtifactPayload(t, indexArtifact)
if indexPayload["collectionCount"] != float64(2) {
t.Fatalf("expected collectionCount=2, got %#v", indexPayload)
}
if indexPayload["videoCount"] != float64(3) {
t.Fatalf("expected videoCount=3, got %#v", indexPayload)
}
validateArtifact := mustArtifact(t, mustNodeExecution(t, completed, "validate"), "dataset-validation.json")
validatePayload := mustArtifactPayload(t, validateArtifact)
if validatePayload["validCollectionCount"] != float64(2) {
t.Fatalf("expected validCollectionCount=2, got %#v", validatePayload)
}
qualityArtifact := mustArtifact(t, mustNodeExecution(t, completed, "quality"), "quality-report.json")
qualityPayload := mustArtifactPayload(t, qualityArtifact)
if qualityPayload["videoCount"] != float64(3) {
t.Fatalf("expected quality videoCount=3, got %#v", qualityPayload)
}
if qualityPayload["passedCount"] != float64(2) {
t.Fatalf("expected passedCount=2, got %#v", qualityPayload)
}
filterArtifact := mustArtifact(t, mustNodeExecution(t, completed, "filter"), "filtered-videos.json")
filterPayload := mustArtifactPayload(t, filterArtifact)
if filterPayload["count"] != float64(1) {
t.Fatalf("expected filter count=1, got %#v", filterPayload)
}
sliceExecution := mustNodeExecution(t, completed, "slice")
clipsArtifact := mustArtifact(t, sliceExecution, "clips-manifest.json")
clipsPayload := mustArtifactPayload(t, clipsArtifact)
if clipsPayload["clipCount"] != float64(2) {
t.Fatalf("expected clipCount=2, got %#v", clipsPayload)
}
// 虚拟切片(VIRTUAL_CUT 默认开启)只记录时间戳区间,不再产出物理片段 MP4;
// 真实裁剪由打包阶段按源视频区间统一物化。
if clipsPayload["virtualCut"] != true {
t.Fatalf("expected clips manifest virtualCut=true, got %#v", clipsPayload["virtualCut"])
}
clipVideos := 0
for _, artifact := range sliceExecution.Artifacts {
if strings.HasSuffix(artifact.Name, ".mp4") {
clipVideos++
}
}
if clipVideos != 0 {
t.Fatalf("expected 0 physical clip videos under virtual cutting, got %d", clipVideos)
}
annotateExecution := mustNodeExecution(t, completed, "annotate")
annotationsArtifact := mustArtifact(t, annotateExecution, "annotations-manifest.json")
annotationsPayload := mustArtifactPayload(t, annotationsArtifact)
if annotationsPayload["annotationCount"] != float64(2) {
t.Fatalf("expected annotationCount=2, got %#v", annotationsPayload)
}
packageExecution := mustNodeExecution(t, completed, "package")
packagesArtifact := mustArtifact(t, packageExecution, "packages-manifest.json")
packagesPayload := mustArtifactPayload(t, packagesArtifact)
if packagesPayload["packageCount"] != float64(2) {
t.Fatalf("expected packageCount=2, got %#v", packagesPayload)
}
packageFiles := 0
for _, artifact := range packageExecution.Artifacts {
if strings.Contains(artifact.Path, "/packages/") && (strings.HasSuffix(artifact.Name, ".json") || strings.HasSuffix(artifact.Name, ".mp4")) {
packageFiles++
}
}
if packageFiles < 8 {
t.Fatalf("expected packaged files to be emitted, got %d artifacts", packageFiles)
}
sampleArtifact := mustArtifact(t, mustNodeExecution(t, completed, "sample"), "audit-samples.json")
samplePayload := mustArtifactPayload(t, sampleArtifact)
if samplePayload["sampledCount"] != float64(2) {
t.Fatalf("expected sampledCount=2, got %#v", samplePayload)
}
}
func TestRuntimeCommitsIncrementalSourceStateOnlyOnSuccessfulRuns(t *testing.T) {
ctx := context.Background()
t.Run("commits state after successful run and next run only emits new files", func(t *testing.T) {
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "增量项目", Description: "incremental"})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "增量源工作流",
Description: "incremental source",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create first run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process first run: %v", err)
}
completedOne, err := dataStore.GetRun(ctx, runOne.ID)
if err != nil {
t.Fatalf("get first run: %v", err)
}
manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json")
manifestOnePayload := mustArtifactPayload(t, manifestOne)
if manifestOnePayload["newCount"] != float64(1) {
t.Fatalf("expected newCount=1 on first run, got %#v", manifestOnePayload)
}
state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestOnePayload["sourceFingerprint"].(string)))
if err != nil {
t.Fatalf("get source state: %v", err)
}
if state == nil || len(state.SeenFingerprints) != 1 {
t.Fatalf("expected committed source state after first run, got %#v", state)
}
writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`)
runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
if err != nil {
t.Fatalf("create second run: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process second run: %v", err)
}
completedTwo, err := dataStore.GetRun(ctx, runTwo.ID)
if err != nil {
t.Fatalf("get second run: %v", err)
}
manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json")
manifestTwoPayload := mustArtifactPayload(t, manifestTwo)
if manifestTwoPayload["listedCount"] != float64(2) {
t.Fatalf("expected listedCount=2 on second run, got %#v", manifestTwoPayload)
}
if manifestTwoPayload["newCount"] != float64(1) {
t.Fatalf("expected newCount=1 on second run, got %#v", manifestTwoPayload)
}
entries, ok := manifestTwoPayload["entries"].([]any)
if !ok || len(entries) != 1 {
t.Fatalf("expected one emitted entry on second run, got %#v", manifestTwoPayload["entries"])
}
entry, ok := entries[0].(map[string]any)
if !ok || !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/meta.json") {
t.Fatalf("expected second run to emit only newly added file, got %#v", entries[0])
}
state, err = dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string)))
if err != nil {
t.Fatalf("get source state after second run: %v", err)
}
if state == nil || len(state.SeenFingerprints) != 2 {
t.Fatalf("expected 2 seen fingerprints after second run, got %#v", state)
}
if state.LastNewCount != 1 || state.LastListedCount != 2 {
t.Fatalf("unexpected source state stats after second run: %#v", state)
}
})
t.Run("directory incremental mode emits entire newly discovered directories", func(t *testing.T) {
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "目录增量项目", Description: "incremental-directory"})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-a.mp4"), "a")
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "目录增量源工作流",
Description: "incremental directory source",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
{Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"},
{Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "1"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create first run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process first run: %v", err)
}
completedOne, err := dataStore.GetRun(ctx, runOne.ID)
if err != nil {
t.Fatalf("get first run: %v", err)
}
manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json")
manifestOnePayload := mustArtifactPayload(t, manifestOne)
if manifestOnePayload["newCount"] != float64(2) {
t.Fatalf("expected first run to emit full batch-a directory, got %#v", manifestOnePayload["newCount"])
}
if manifestOnePayload["incrementalUnit"] != "DIRECTORY" {
t.Fatalf("expected directory incremental unit, got %#v", manifestOnePayload["incrementalUnit"])
}
writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-b.mp4"), "b")
writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`)
writeFile(t, filepath.Join(sourceRoot, "batch-b", "DJI_001", "clip-c.mp4"), "c")
runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
if err != nil {
t.Fatalf("create second run: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process second run: %v", err)
}
completedTwo, err := dataStore.GetRun(ctx, runTwo.ID)
if err != nil {
t.Fatalf("get second run: %v", err)
}
manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json")
manifestTwoPayload := mustArtifactPayload(t, manifestTwo)
if manifestTwoPayload["listedCount"] != float64(5) {
t.Fatalf("expected listedCount=5 on second run, got %#v", manifestTwoPayload["listedCount"])
}
if manifestTwoPayload["newCount"] != float64(2) {
t.Fatalf("expected newCount=2 on second run, got %#v", manifestTwoPayload["newCount"])
}
entries, ok := manifestTwoPayload["entries"].([]any)
if !ok || len(entries) != 2 {
t.Fatalf("expected two emitted entries for new directory, got %#v", manifestTwoPayload["entries"])
}
for _, rawEntry := range entries {
entry, ok := rawEntry.(map[string]any)
if !ok {
t.Fatalf("unexpected entry payload: %#v", rawEntry)
}
if !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/") {
t.Fatalf("expected second run to emit only newly discovered directory contents, got %#v", entry["path"])
}
}
state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string)))
if err != nil {
t.Fatalf("get source state after second run: %v", err)
}
if state == nil || len(state.SeenFingerprints) != 2 {
t.Fatalf("expected two seen directory keys after second run, got %#v", state)
}
if state.IncrementalUnit != "DIRECTORY" || state.DirectoryDepth != 1 {
t.Fatalf("expected committed directory incremental state, got %#v", state)
}
})
t.Run("does not commit state when downstream run fails", func(t *testing.T) {
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "失败增量项目", Description: "incremental-fail"})
if err != nil {
t.Fatalf("create project: %v", err)
}
sourceRoot := t.TempDir()
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "增量失败工作流",
Description: "incremental fail",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "LOCAL"},
{Key: "SOURCE_PATH", Value: sourceRoot},
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "fail",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "fail",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "echo boom >&2; exit 1"}},
},
Position: domain.Position{X: 240, Y: 0},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-fail", Source: "source", SourceHandle: "out-1", Target: "fail", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process failed run: %v", err)
}
failedRun, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get failed run: %v", err)
}
if failedRun == nil || failedRun.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %#v", failedRun)
}
manifest := mustArtifact(t, mustNodeExecution(t, failedRun, "source"), "source-manifest.json")
manifestPayload := mustArtifactPayload(t, manifest)
state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestPayload["sourceFingerprint"].(string)))
if err != nil {
t.Fatalf("get source state after failed run: %v", err)
}
if state != nil {
t.Fatalf("expected no committed source state after failed run, got %#v", state)
}
})
}
func TestInputStorageConnectionIDUsesDirectoryReferenceAndRejectsAmbiguity(t *testing.T) {
payload := domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeFile, Ref: &domain.ArtifactReference{ConnectionID: "storage-source", Bucket: "bucket", Prefix: "capture", Kind: domain.ArtifactRefKindDirectory}}
connectionID, err := inputStorageConnectionID([]domain.NodeInputBinding{{TargetPortID: "in-1", Payload: &payload}})
if err != nil || connectionID != "storage-source" {
t.Fatalf("connectionID=%q err=%v", connectionID, err)
}
other := payload
refCopy := *payload.Ref
refCopy.ConnectionID = "storage-other"
other.Ref = &refCopy
if _, err := inputStorageConnectionID([]domain.NodeInputBinding{{Payload: &payload}, {Payload: &other}}); err == nil {
t.Fatal("multiple input storage connections must fail closed")
}
}
func TestPrepareExecutionNodeResolvesStorageConnectionSecrets(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 主连接",
Provider: domain.StorageProviderBOS,
Endpoint: "https://gz.bcebos.com",
AccessKeyID: "ak-runtime-1234",
SecretAccessKey: "sk-runtime-5678",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
workflow := &domain.WorkflowRecord{
ID: "workflow-scan",
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
ConnectionID: connection.ID,
Bucket: "datastorage",
Prefix: "algo/X-platform-test",
ScanMode: "INCREMENTAL",
},
}
node := domain.WorkflowNode{
ID: "node-bos-scan-runtime",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "BOS 增量扫描",
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "BOS"},
},
},
}
prepared, err := runtime.prepareExecutionNode(ctx, workflow, node)
if err != nil {
t.Fatalf("prepare execution node: %v", err)
}
params := nodeHyperparamsMap(prepared)
if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" {
t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"])
}
if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-5678" {
t.Fatalf("expected resolved bos secrets, got %#v", params)
}
if params["SOURCE_PATH"] != "bos://datastorage/algo/X-platform-test" {
t.Fatalf("expected source path from bucket/prefix, got %q", params["SOURCE_PATH"])
}
}
func TestPrepareExecutionNodeUsesRuntimeGlobalBOSEndpointForConnectionWithoutEndpoint(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 全局 endpoint 连接",
Provider: domain.StorageProviderBOS,
AccessKeyID: "ak-runtime-default",
SecretAccessKey: "sk-runtime-default",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
runtime.globalBOSEndpoint = "https://gz.bcebos.com"
workflow := &domain.WorkflowRecord{
ID: "workflow-default-bos",
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
ConnectionID: connection.ID,
Bucket: "datastorage",
Prefix: "algo/default-endpoint",
ScanMode: "INCREMENTAL",
},
}
node := domain.WorkflowNode{
ID: "node-default-bos",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "BOS 增量扫描",
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "BOS"}},
},
}
prepared, err := runtime.prepareExecutionNode(ctx, workflow, node)
if err != nil {
t.Fatalf("prepare execution node: %v", err)
}
params := nodeHyperparamsMap(prepared)
if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" {
t.Fatalf("expected runtime global bos endpoint, got %q", params["BOS_ENDPOINT"])
}
if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-default" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-default" {
t.Fatalf("expected resolved bos secrets, got %#v", params)
}
}
func TestPrepareExecutionNodeKeepsEmptyEndpointWithoutGlobalBOSFallback(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 无 endpoint 连接",
Provider: domain.StorageProviderBOS,
AccessKeyID: "ak-runtime-empty-endpoint",
SecretAccessKey: "sk-runtime-empty-endpoint",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
workflow := &domain.WorkflowRecord{
ID: "workflow-empty-bos-endpoint",
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
ConnectionID: connection.ID,
Bucket: "datastorage",
Prefix: "algo/empty-endpoint",
ScanMode: "INCREMENTAL",
},
}
node := domain.WorkflowNode{
ID: "node-empty-bos-endpoint",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "BOS 增量扫描",
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "BOS"}},
},
}
prepared, err := runtime.prepareExecutionNode(ctx, workflow, node)
if err != nil {
t.Fatalf("prepare execution node: %v", err)
}
params := nodeHyperparamsMap(prepared)
if params["BOS_ENDPOINT"] != "" {
t.Fatalf("expected empty endpoint without global fallback, got %q", params["BOS_ENDPOINT"])
}
if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-empty-endpoint" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-empty-endpoint" {
t.Fatalf("expected resolved bos secrets, got %#v", params)
}
}
func TestPrepareExecutionNodeResolvesNodeLevelStorageConnectionSecrets(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 节点连接",
Provider: domain.StorageProviderBOS,
Endpoint: "https://su.bcebos.com",
AccessKeyID: "ak-node-1234",
SecretAccessKey: "sk-node-5678",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
node := domain.WorkflowNode{
ID: "node-oss-source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "OSS 数据源",
ConnectionID: connection.ID,
Bucket: "datastorage",
Prefix: "factory/output",
Hyperparams: []domain.HyperParam{
{Key: "SOURCE_PATH", Value: ""},
{Key: "DATA_SOURCE", Value: "BOS"},
},
},
}
prepared, err := runtime.prepareExecutionNode(ctx, nil, node)
if err != nil {
t.Fatalf("prepare execution node: %v", err)
}
params := nodeHyperparamsMap(prepared)
if params["DATA_SOURCE"] != "BOS" {
t.Fatalf("expected BOS data source, got %q", params["DATA_SOURCE"])
}
if params["BOS_ENDPOINT"] != "https://su.bcebos.com" {
t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"])
}
if params["BOS_ACCESS_KEY_ID"] != "ak-node-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-node-5678" {
t.Fatalf("expected resolved node bos secrets, got %#v", params)
}
if params["SOURCE_PATH"] != "bos://datastorage/factory/output" {
t.Fatalf("expected source path from node bucket/prefix, got %q", params["SOURCE_PATH"])
}
}
func TestPrepareExecutionNodeResolvesMongoConnectionSecrets(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{
Name: "业务 MongoDB",
URI: "mongodb://business-mongo:27017",
Database: "business_data",
})
if err != nil {
t.Fatalf("create mongo connection: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
node := domain.WorkflowNode{
ID: "node-mongo-write",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "MongoDB 写入",
Hyperparams: []domain.HyperParam{
{Key: "MONGO_CONNECTION_ID", Value: connection.ID},
{Key: "MONGO_COLLECTION", Value: "qualityReports"},
},
},
}
prepared, err := runtime.prepareExecutionNode(ctx, nil, node)
if err != nil {
t.Fatalf("prepare execution node: %v", err)
}
params := nodeHyperparamsMap(prepared)
if params["MONGO_URI"] != "mongodb://business-mongo:27017" {
t.Fatalf("expected resolved mongo uri, got %q", params["MONGO_URI"])
}
if params["MONGO_DB"] != "business_data" {
t.Fatalf("expected resolved mongo db, got %q", params["MONGO_DB"])
}
if params["MONGO_TARGET"] != "business" {
t.Fatalf("expected mongo target business, got %q", params["MONGO_TARGET"])
}
}
func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 2
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Dispatch 项目", Description: "dispatch"})
if err != nil {
t.Fatalf("create project: %v", err)
}
createWorkflow := func(name string) *domain.WorkflowRecord {
t.Helper()
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: name,
Description: "dispatch test",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow %s: %v", name, err)
}
return workflow
}
workflowA := createWorkflow("并发工作流 A")
workflowB := createWorkflow("并发工作流 B")
runA, err := dataStore.CreateRun(ctx, workflowA.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run A: %v", err)
}
runB, err := dataStore.CreateRun(ctx, workflowB.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run B: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
runs, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatalf("list runs: %v", err)
}
runningCount := 0
for _, run := range runs {
if run.Status == domain.RunStatusRunning {
runningCount++
}
}
if runningCount != 2 {
t.Fatalf("expected 2 running runs after dispatch, got %d", runningCount)
}
waitForRunStatus(t, ctx, dataStore, runA.ID, domain.RunStatusCompleted, 8*time.Second)
waitForRunStatus(t, ctx, dataStore, runB.ID, domain.RunStatusCompleted, 8*time.Second)
}
func TestSelectQueuedRunsLetsNewCapturePassAuditWaits(t *testing.T) {
settings := &domain.SettingsRecord{}
settings.Worker.MaxConcurrentRuns = 2
workflows := []domain.WorkflowRecord{
{ID: "wf-human", MaxConcurrentRuns: 1},
{ID: "wf-data"},
{ID: "wf-video"},
}
runs := []domain.RunRecord{
{ID: "audit-wait", WorkflowID: "wf-human", Status: domain.RunStatusWaitingForAudit},
{ID: "human-next", WorkflowID: "wf-human", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:01Z"},
{ID: "data-next", WorkflowID: "wf-data", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:02Z"},
{ID: "video-next", WorkflowID: "wf-video", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:03Z"},
}
selected, queueDepth, activeTotal := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
if activeTotal != 0 {
t.Fatalf("waiting_for_audit should not consume global dispatch slots, activeTotal=%d", activeTotal)
}
if queueDepth != 3 {
t.Fatalf("queueDepth = %d, want 3", queueDepth)
}
if len(selected) != 2 {
t.Fatalf("selected = %#v, want two runs", selected)
}
if selected[0] != "human-next" || selected[1] != "data-next" {
t.Fatalf("selected = %#v, want the next Capture plus the next workflow in FIFO order", selected)
}
}
func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 3
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Override 项目", Description: "override"})
if err != nil {
t.Fatalf("create project: %v", err)
}
limitedWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "受限工作流",
Description: "limited",
Type: domain.WorkflowTypeBatch,
MaxConcurrentRuns: 1,
Nodes: []domain.WorkflowNode{{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create limited workflow: %v", err)
}
freeWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "自由工作流",
Description: "free",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
}},
})
if err != nil {
t.Fatalf("create free workflow: %v", err)
}
runLimitedA, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create limited run A: %v", err)
}
runLimitedB, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create limited run B: %v", err)
}
runFree, err := dataStore.CreateRun(ctx, freeWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create free run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
limitedRunAState, err := dataStore.GetRun(ctx, runLimitedA.ID)
if err != nil {
t.Fatalf("get limited run A: %v", err)
}
limitedRunBState, err := dataStore.GetRun(ctx, runLimitedB.ID)
if err != nil {
t.Fatalf("get limited run B: %v", err)
}
freeRunState, err := dataStore.GetRun(ctx, runFree.ID)
if err != nil {
t.Fatalf("get free run: %v", err)
}
if limitedRunAState.Status != domain.RunStatusRunning {
t.Fatalf("expected first limited run to be running, got %s", limitedRunAState.Status)
}
if limitedRunBState.Status != domain.RunStatusQueued {
t.Fatalf("expected second limited run to remain queued, got %s", limitedRunBState.Status)
}
if freeRunState.Status != domain.RunStatusRunning {
t.Fatalf("expected free workflow run to be running, got %s", freeRunState.Status)
}
waitForRunStatus(t, ctx, dataStore, runLimitedA.ID, domain.RunStatusCompleted, 8*time.Second)
waitForRunStatus(t, ctx, dataStore, runFree.ID, domain.RunStatusCompleted, 8*time.Second)
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available second pass: %v", err)
}
waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second)
}
func TestRuntimeProcessAvailableSkipsStreamingWorkflowWithMissingProject(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "有效项目", Description: "valid"})
if err != nil {
t.Fatalf("create project: %v", err)
}
validWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "正常工作流",
Description: "valid workflow",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "command",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "command",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create valid workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, validWorkflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create queued run: %v", err)
}
orphanWorkflow := domain.WorkflowRecord{
ID: "workflow-orphan-streaming",
ProjectID: "project-missing",
Name: "孤儿增量工作流",
Description: "orphan streaming workflow",
Type: domain.WorkflowTypeStreaming,
Schedule: &domain.WorkflowSchedule{Interval: 30},
Nodes: []domain.WorkflowNode{},
Edges: []domain.WorkflowEdge{},
Status: domain.WorkflowStatusDraft,
CreatedAt: nowISO(),
UpdatedAt: nowISO(),
}
if err := dataStore.UpsertWorkflow(ctx, orphanWorkflow); err != nil {
t.Fatalf("upsert orphan workflow: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available should skip orphan workflow instead of failing: %v", err)
}
waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
}
func TestShouldEnqueueScheduledRunEveryHoursUsesAnchorTime(t *testing.T) {
workflow := domain.WorkflowRecord{
ID: "workflow-hours",
Type: domain.WorkflowTypeStreaming,
Schedule: &domain.WorkflowSchedule{
Mode: domain.WorkflowScheduleModeEveryHours,
IntervalHours: 4,
AnchorAt: "2026-04-21T10:15:00Z",
},
}
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 15, 0, 0, time.UTC)) {
t.Fatalf("expected first hourly window to enqueue at anchor time")
}
workflow.LastRunAt = "2026-04-21T10:15:00Z"
if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 14, 59, 0, time.UTC)) {
t.Fatalf("expected hourly schedule to wait until full interval elapsed")
}
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 15, 0, 0, time.UTC)) {
t.Fatalf("expected hourly schedule to enqueue on the next anchored slot")
}
}
func TestShouldEnqueueScheduledRunEveryDaysAtFixedTimeUsesAnchorDate(t *testing.T) {
workflow := domain.WorkflowRecord{
ID: "workflow-days",
Type: domain.WorkflowTypeStreaming,
Schedule: &domain.WorkflowSchedule{
Mode: domain.WorkflowScheduleModeEveryDaysAtTime,
DayInterval: 2,
TimeOfDay: "18:30",
Timezone: "Asia/Shanghai",
AnchorAt: "2026-04-21T09:00:00Z",
},
}
if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 29, 59, 0, time.UTC)) {
t.Fatalf("expected daily schedule to wait until configured local time")
}
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 30, 0, 0, time.UTC)) {
t.Fatalf("expected daily schedule to enqueue at configured local time")
}
workflow.LastRunAt = "2026-04-21T10:30:00Z"
if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 29, 59, 0, time.UTC)) {
t.Fatalf("expected next two-day slot to remain pending before local HH:mm")
}
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 30, 0, 0, time.UTC)) {
t.Fatalf("expected next two-day slot to enqueue at configured local HH:mm")
}
}
func TestRuntimeProcessRunResumesQueuedPartialRunFromCompletedDependencies(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "恢复工作流",
Description: "resume",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}},
},
},
{
ID: "transform",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "JSON 映射重命名",
Category: "DATA",
Description: "transform",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "FROM_KEY", Value: "items"}, {Key: "TO_KEY", Value: "entries"}},
},
},
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "sink",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}},
},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-transform", Source: "source", SourceHandle: "out-1", Target: "transform", TargetHandle: "in-1"},
{ID: "transform-upload", Source: "transform", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
run.Status = domain.RunStatusQueued
run.CurrentNodeID = ""
run.NodeExecutions = []domain.NodeExecutionRecord{
{
NodeID: "source",
NodeLabel: "真实数据源",
Status: domain.NodeExecutionStatusCompleted,
StartedAt: nowISO(),
EndedAt: nowISO(),
Logs: []domain.LogEntryRecord{},
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-source", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "payload.json", Type: domain.PortTypeJSON, Path: filepath.Join(t.TempDir(), "payload.json"), Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1}}},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
},
}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
if execution := mustNodeExecution(t, completed, "transform"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected transform to complete after resume, got %s", execution.Status)
}
if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected upload to complete after resume, got %s", execution.Status)
}
}
func TestRuntimeReconcileActiveRunsRequeuesClaimedRunWithoutJobs(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "僵尸运行恢复",
Description: "recover",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
run.Status = domain.RunStatusRunning
run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano)
run.UpdatedAt = run.StartedAt
run.NodeExecutions = []domain.NodeExecutionRecord{}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile active runs: %v", err)
}
recovered, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get recovered run: %v", err)
}
if recovered == nil || recovered.Status != domain.RunStatusQueued {
t.Fatalf("expected run to be re-queued, got %#v", recovered)
}
if recovered.StartedAt != "" {
t.Fatalf("expected startedAt cleared after requeue, got %q", recovered.StartedAt)
}
}
func TestRuntimeReconcileActiveRunsClearsQueuedRunWithMissingActiveJob(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover queued active"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "队列运行节点恢复",
Description: "recover queued active",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
startedAt := time.Now().UTC().Add(-10 * time.Minute).Format(time.RFC3339Nano)
run.Status = domain.RunStatusQueued
run.CurrentNodeID = "source"
run.StartedAt = startedAt
run.UpdatedAt = startedAt
run.NodeExecutions = []domain.NodeExecutionRecord{{
NodeID: "source",
NodeLabel: "真实数据源",
Status: domain.NodeExecutionStatusRunning,
StartedAt: startedAt,
Logs: []domain.LogEntryRecord{{Timestamp: startedAt, Level: "INFO", Message: "Starting node execution", Operator: "真实数据源"}},
}}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile active runs: %v", err)
}
recovered, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get recovered run: %v", err)
}
if recovered == nil || recovered.Status != domain.RunStatusQueued {
t.Fatalf("expected run to remain queued, got %#v", recovered)
}
if recovered.CurrentNodeID != "" {
t.Fatalf("expected current node cleared, got %q", recovered.CurrentNodeID)
}
if len(recovered.NodeExecutions) != 1 || recovered.NodeExecutions[0].Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected stale source execution retained as failed recovery evidence, got %#v", recovered.NodeExecutions)
}
if !strings.Contains(recovered.NodeExecutions[0].Error, "recovery attempt 1/3") {
t.Fatalf("expected recovery attempt evidence, got %#v", recovered.NodeExecutions[0])
}
// Recovery deliberately applies a short backoff in production. Make this
// fixture dispatchable now so the test can verify the resumed execution.
recovered.AvailableAt = nowISO()
if err := dataStore.UpsertRun(ctx, *recovered); err != nil {
t.Fatalf("make recovered run dispatchable: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once after recovery: %v", err)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
if execution := mustNodeExecution(t, completed, "source"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected source to complete after stale execution recovery, got %s", execution.Status)
}
}
func TestRuntimeProcessRunResumesConnectedPendingNodes(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume connected"})
if err != nil {
t.Fatalf("create project: %v", err)
}
nodes := []domain.WorkflowNode{
{ID: "source-a", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"a\"}"}}}},
{ID: "source-b", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"b\"}"}}}},
{ID: "source-c", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"c\"}"}}}},
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "无边批次恢复",
Description: "resume connected",
Type: domain.WorkflowTypeBatch,
Nodes: nodes,
Edges: []domain.WorkflowEdge{
{ID: "source-a-to-source-b", Source: "source-a", SourceHandle: "out-1", Target: "source-b", TargetHandle: "in-1"},
{ID: "source-b-to-source-c", Source: "source-b", SourceHandle: "out-1", Target: "source-c", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
startedAt := nowISO()
run.Status = domain.RunStatusQueued
run.CurrentNodeID = ""
run.NodeExecutions = []domain.NodeExecutionRecord{{
NodeID: "source-a",
NodeLabel: "真实数据源",
Status: domain.NodeExecutionStatusCompleted,
StartedAt: startedAt,
EndedAt: startedAt,
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-a", RunID: run.ID, NodeID: "source-a", PortID: "out-1", Name: "a.json", Type: domain.PortTypeJSON, Data: map[string]any{"name": "a"}}},
}}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
if len(completed.NodeExecutions) != 3 {
t.Fatalf("expected all unconnected nodes executed, got %#v", completed.NodeExecutions)
}
for _, node := range nodes {
if execution := mustNodeExecution(t, completed, node.ID); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected %s completed, got %s", node.ID, execution.Status)
}
}
}
func TestRuntimeReconcileActiveRunsRecoversSucceededNodeAndResumesDownstream(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover success"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成功节点恢复",
Description: "recover success",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}},
},
},
{
ID: "annotate",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "视频标注",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{{Key: "MODEL", Value: "stub"}},
},
},
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}},
},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-annotate", Source: "source", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"},
{ID: "annotate-upload", Source: "annotate", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
sourceArtifact := domain.ExecutionArtifact{
ID: "artifact-source",
RunID: run.ID,
NodeID: "source",
PortID: "out-1",
Name: "payload.json",
Type: domain.PortTypeJSON,
Path: filepath.Join(t.TempDir(), "payload.json"),
Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1},
}
run.Status = domain.RunStatusRunning
run.CurrentNodeID = "annotate"
run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano)
run.UpdatedAt = run.StartedAt
run.NodeExecutions = []domain.NodeExecutionRecord{
{
NodeID: "source",
NodeLabel: "真实数据源",
Status: domain.NodeExecutionStatusCompleted,
StartedAt: run.StartedAt,
EndedAt: run.StartedAt,
Logs: []domain.LogEntryRecord{},
Artifacts: []domain.ExecutionArtifact{sourceArtifact},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
},
{
NodeID: "annotate",
NodeLabel: "视频标注",
Status: domain.NodeExecutionStatusRunning,
StartedAt: run.StartedAt,
Logs: []domain.LogEntryRecord{{Timestamp: run.StartedAt, Level: "INFO", Message: "Starting node execution", Operator: "视频标注"}},
Artifacts: []domain.ExecutionArtifact{},
DataRefs: []domain.DataRefRecord{},
ResourceSeries: []domain.ResourceSample{},
},
}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
launcher := testLauncher{
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
env := append([]string{}, os.Environ()...)
keys := make([]string, 0, len(input.Env))
for key := range input.Env {
keys = append(keys, key)
}
sort.Strings(keys)
env = append(env, "RUN_ID="+input.RunID, "WORKFLOW_ID="+input.WorkflowID, "NODE_ID="+input.NodeID, "NODE_LABEL="+input.NodeLabel)
for _, key := range keys {
env = append(env, key+"="+input.Env[key])
}
cmd := exec.CommandContext(ctx, "python3", operatorRunnerPath(t))
cmd.Env = env
output, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("%w: %s", err, string(output))
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID), Namespace: "droboticflow"},
Output: string(output),
}, nil
},
inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) {
if runID == run.ID && nodeID == "annotate" {
return &k8s.JobInspection{
Reference: &k8s.JobReference{JobName: "job-annotate", Namespace: "droboticflow"},
State: k8s.JobStateSucceeded,
Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"annotation-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"annotate\",\"portId\":\"out-1\",\"name\":\"annotation.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/annotation.json\",\"data\":{\"items\":[{\"label\":\"walking\"}],\"count\":1}}],\"logs\":[],\"resourceSeries\":[]}",
}, nil
}
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
},
}
runtime := newTestRuntimeWithLauncher(t, dataStore, launcher)
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile active runs: %v", err)
}
recovered, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get recovered run: %v", err)
}
if recovered == nil || recovered.Status != domain.RunStatusQueued {
t.Fatalf("expected recovered run queued, got %#v", recovered)
}
if execution := mustNodeExecution(t, recovered, "annotate"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected annotate completed after recovery, got %s", execution.Status)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once after recovery: %v", err)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected downstream upload completed, got %s", execution.Status)
}
}
func TestRuntimeReconcileActiveRunsCompletesRunWhenRecoveredFinalNodeSucceeded(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复完成项目", Description: "recover final success"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "最终节点恢复",
Description: "recover final success",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
},
},
{
ID: "sample",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "抽检任务",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}},
},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-sample", Source: "source", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
startedAt := time.Now().UTC().Add(-3 * time.Minute).Format(time.RFC3339Nano)
run.Status = domain.RunStatusRunning
run.CurrentNodeID = "sample"
run.StartedAt = startedAt
run.UpdatedAt = startedAt
run.NodeExecutions = []domain.NodeExecutionRecord{
{
NodeID: "source",
NodeLabel: "真实数据源",
Status: domain.NodeExecutionStatusCompleted,
StartedAt: startedAt,
EndedAt: startedAt,
Artifacts: []domain.ExecutionArtifact{{
ID: "artifact-source",
RunID: run.ID,
NodeID: "source",
PortID: "out-1",
Name: "payload.json",
Type: domain.PortTypeJSON,
Data: map[string]any{"ok": true},
}},
},
{
NodeID: "sample",
NodeLabel: "抽检任务",
Status: domain.NodeExecutionStatusRunning,
StartedAt: startedAt,
Logs: []domain.LogEntryRecord{{Timestamp: startedAt, Level: "INFO", Message: "Starting node execution", Operator: "抽检任务"}},
},
}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) {
if runID == run.ID && nodeID == "sample" {
return &k8s.JobInspection{
Reference: &k8s.JobReference{JobName: "job-sample", Namespace: "droboticflow"},
Image: "registry.example/operator@sha256:" + strings.Repeat("d", 64),
State: k8s.JobStateSucceeded,
Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"sample-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"sample\",\"portId\":\"out-1\",\"name\":\"audit-samples.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/audit-samples.json\",\"data\":{\"sampledCount\":1}}],\"logs\":[],\"resourceSeries\":[]}",
}, nil
}
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
},
})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile active runs: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get completed run: %v", err)
}
if completed == nil || completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected recovered final node to complete run, got %#v", completed)
}
if completed.CurrentNodeID != "" {
t.Fatalf("expected current node cleared, got %q", completed.CurrentNodeID)
}
if execution := mustNodeExecution(t, completed, "sample"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected sample completed, got %s", execution.Status)
}
if execution := mustNodeExecution(t, completed, "sample"); execution.OperatorImageID != "registry.example/operator@sha256:"+strings.Repeat("d", 64) {
t.Fatalf("recovery lost actual Job digest: %s", execution.OperatorImageID)
}
finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
if err != nil {
t.Fatalf("get workflow: %v", err)
}
if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted {
t.Fatalf("expected workflow completed, got %#v", finalWorkflow)
}
}
func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 3
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Parallel DAG 项目", Description: "parallel dag"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "并行分支工作流",
Description: "parallel dag",
Type: domain.WorkflowTypeBatch,
MaxConcurrentRuns: 3,
Nodes: []domain.WorkflowNode{
{
ID: "source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "OSS 数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "INLINE"},
{Key: "SOURCE_TEXT", Value: "parallel"},
},
},
Position: domain.Position{X: 0, Y: 0},
},
{
ID: "branch-a",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "branch a",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-a", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"a\"}' > result-a.json"}},
},
Position: domain.Position{X: 200, Y: -120},
},
{
ID: "branch-b",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "branch b",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-b", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"b\"}' > result-b.json"}},
},
Position: domain.Position{X: 200, Y: 0},
},
{
ID: "branch-c",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "branch c",
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-c", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"c\"}' > result-c.json"}},
},
Position: domain.Position{X: 200, Y: 120},
},
},
Edges: []domain.WorkflowEdge{
{ID: "source-a", Source: "source", SourceHandle: "out-1", Target: "branch-a", TargetHandle: "in-1"},
{ID: "source-b", Source: "source", SourceHandle: "out-1", Target: "branch-b", TargetHandle: "in-1"},
{ID: "source-c", Source: "source", SourceHandle: "out-1", Target: "branch-c", TargetHandle: "in-1"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
runningRun := waitForRunningNodeCount(t, ctx, dataStore, run.ID, 3, 5*time.Second)
if runningRun.Status != domain.RunStatusRunning {
t.Fatalf("expected run to be running while branches execute, got %s", runningRun.Status)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second)
for _, nodeID := range []string{"branch-a", "branch-b", "branch-c"} {
if execution := mustNodeExecution(t, completed, nodeID); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected node %s to complete, got %s", nodeID, execution.Status)
}
}
}
func TestRuntimeProcessRunContinuesConnectedBranchesPastConcurrencyLimit(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.MaxConcurrentRuns = 2
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Disconnected 项目", Description: "disconnected nodes"})
if err != nil {
t.Fatalf("create project: %v", err)
}
nodes := []domain.WorkflowNode{{
ID: "root",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "root node",
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"node":"root"}' > root.json`}},
},
}}
edges := make([]domain.WorkflowEdge, 0, 5)
for index := 0; index < 5; index++ {
nodeID := fmt.Sprintf("independent-%d", index)
nodes = append(nodes, domain.WorkflowNode{
ID: nodeID,
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "independent node",
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: fmt.Sprintf(`printf '{"node":"%s"}' > %s.json`, nodeID, nodeID)}},
},
})
edges = append(edges, domain.WorkflowEdge{ID: "root-to-" + nodeID, Source: "root", Target: nodeID})
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "单连通分支工作流",
Description: "connected branch batch",
Type: domain.WorkflowTypeBatch,
MaxConcurrentRuns: 2,
Nodes: nodes,
Edges: edges,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second)
if len(completed.NodeExecutions) != len(nodes) {
t.Fatalf("expected %d executed nodes, got %d: %#v", len(nodes), len(completed.NodeExecutions), completed.NodeExecutions)
}
for _, node := range nodes {
if execution := mustNodeExecution(t, completed, node.ID); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected node %s to complete, got %s", node.ID, execution.Status)
}
}
}
func TestRuntimeRerunSubtreeOnlyExecutesSelectedSubgraph(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "子链路重跑项目", Description: "rerun subtree"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "子链路重跑工作流",
Description: "rerun subtree",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "root",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Description: "root",
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"root":true}' > root.json`}},
},
},
{
ID: "start-a",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Description: "branch a start",
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a"}' > branch-a.json`}},
},
},
{
ID: "downstream-a",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Description: "branch a end",
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a-downstream"}' > branch-a-downstream.json`}},
},
},
{
ID: "start-b",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Description: "branch b start",
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"b"}' > branch-b.json`}},
},
},
},
Edges: []domain.WorkflowEdge{
{ID: "root-to-start-a", Source: "root", Target: "start-a"},
{ID: "root-to-start-b", Source: "root", Target: "start-b"},
{ID: "a-to-downstream", Source: "start-a", Target: "downstream-a"},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
run.Trigger = domain.RunTriggerRerunSubtree
run.RestartFromNodeID = "start-a"
startedAt := nowISO()
run.NodeExecutions = []domain.NodeExecutionRecord{{
NodeID: "root",
NodeLabel: "Shell 命令执行",
Status: domain.NodeExecutionStatusCompleted,
StartedAt: startedAt,
EndedAt: startedAt,
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-root", RunID: run.ID, NodeID: "root", PortID: "out-1", Name: "root.json", Type: domain.PortTypeJSON, Data: map[string]any{"root": true}}},
}}
run.Status = domain.RunStatusQueued
run.UpdatedAt = nowISO()
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert rerun subtree run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
if execution := mustNodeExecution(t, completed, "start-a"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected start-a to complete, got %s", execution.Status)
}
if execution := mustNodeExecution(t, completed, "downstream-a"); execution.Status != domain.NodeExecutionStatusCompleted {
t.Fatalf("expected downstream-a to complete, got %s", execution.Status)
}
for _, execution := range completed.NodeExecutions {
if execution.NodeID == "start-b" {
t.Fatalf("expected unrelated branch start-b not to execute, got %#v", execution)
}
}
}
func TestProcessRunKeepsClaimedRunRunningWhenExistingNodeExecutionIsActive(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
runtime := &Runtime{
store: dataStore,
operatorRunnerImage: "runner:latest",
workspaceRoot: "/workspace",
launcher: testLauncher{},
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: "",
Name: "resume-active-node",
Description: "resume active execution",
Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source"}}},
Edges: []domain.WorkflowEdge{},
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
Schedule: nil,
ScanConfig: nil,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
run.Status = domain.RunStatusQueued
run.CurrentNodeID = "source"
run.NodeExecutions = []domain.NodeExecutionRecord{
{
NodeID: "source",
NodeLabel: "Source",
Status: domain.NodeExecutionStatusRunning,
StartedAt: nowISO(),
},
}
run.UpdatedAt = nowISO()
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
processed := mustProcessOnceRun(t, runtime, ctx)
if processed.Status != domain.RunStatusRunning {
t.Fatalf("expected running status, got %s", processed.Status)
}
if processed.CurrentNodeID != "source" {
t.Fatalf("expected currentNodeID=source, got %s", processed.CurrentNodeID)
}
if execution := mustNodeExecution(t, processed, "source"); execution.Status != domain.NodeExecutionStatusRunning {
t.Fatalf("expected source execution running, got %s", execution.Status)
}
}
func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Timeout 项目", Description: "timeout"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "超时工作流",
Description: "timeout",
Type: domain.WorkflowTypeBatch,
RunTimeoutSeconds: 1,
Nodes: []domain.WorkflowNode{
{
ID: "slow-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "slow",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
failed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get failed run: %v", err)
}
if failed == nil || failed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %#v", failed)
}
if !strings.Contains(failed.Error, "超时") {
t.Fatalf("expected timeout error, got %q", failed.Error)
}
execution := mustNodeExecution(t, failed, "slow-node")
if execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed node execution, got %s", execution.Status)
}
storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
if err != nil {
t.Fatalf("get workflow: %v", err)
}
if storedWorkflow == nil || storedWorkflow.Status != domain.WorkflowStatusFailed {
t.Fatalf("expected failed workflow, got %#v", storedWorkflow)
}
}
func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
markerPath := filepath.Join(t.TempDir(), "retry-marker")
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry 项目", Description: "retry"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "自动重试工作流",
Description: "retry",
Type: domain.WorkflowTypeBatch,
MaxRetryAttempts: 1,
Nodes: []domain.WorkflowNode{
{
ID: "flaky-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "flaky",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{
Key: "COMMAND",
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create initial run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("first process once: %v", err)
}
firstAttempt, err := dataStore.GetRun(ctx, initialRun.ID)
if err != nil {
t.Fatalf("get first attempt: %v", err)
}
if firstAttempt == nil || firstAttempt.Status != domain.RunStatusFailed {
t.Fatalf("expected first attempt to fail, got %#v", firstAttempt)
}
if firstAttempt.Attempt != 1 {
t.Fatalf("expected first attempt number to be 1, got %d", firstAttempt.Attempt)
}
runs, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 2 {
t.Fatalf("expected original + retry run, got %d", len(runs))
}
var retryRun domain.RunRecord
foundRetry := false
for _, candidate := range runs {
if candidate.ID == initialRun.ID {
continue
}
retryRun = candidate
foundRetry = true
break
}
if !foundRetry {
t.Fatalf("expected retry run to be created")
}
if retryRun.Trigger != domain.RunTriggerRetry {
t.Fatalf("expected retry trigger, got %s", retryRun.Trigger)
}
if retryRun.Attempt != 2 {
t.Fatalf("expected retry attempt number to be 2, got %d", retryRun.Attempt)
}
if retryRun.ParentRunID != initialRun.ID {
t.Fatalf("expected retry parent run %s, got %s", initialRun.ID, retryRun.ParentRunID)
}
if retryRun.Status != domain.RunStatusQueued {
t.Fatalf("expected retry run to be queued, got %s", retryRun.Status)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("second process once: %v", err)
}
completedRetry, err := dataStore.GetRun(ctx, retryRun.ID)
if err != nil {
t.Fatalf("get retry run: %v", err)
}
if completedRetry == nil || completedRetry.Status != domain.RunStatusCompleted {
t.Fatalf("expected retry run to complete, got %#v", completedRetry)
}
finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
if err != nil {
t.Fatalf("get final workflow: %v", err)
}
if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted {
t.Fatalf("expected workflow to recover and complete, got %#v", finalWorkflow)
}
}
func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Node Timeout 项目", Description: "node timeout"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "节点超时工作流",
Description: "node timeout",
Type: domain.WorkflowTypeBatch,
NodeTimeoutSeconds: 1,
Nodes: []domain.WorkflowNode{
{
ID: "slow-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "slow",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("process once: %v", err)
}
failed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get failed run: %v", err)
}
if failed == nil || failed.Status != domain.RunStatusFailed {
t.Fatalf("expected failed run, got %#v", failed)
}
if !strings.Contains(failed.Error, "节点执行超时") {
t.Fatalf("expected node timeout error, got %q", failed.Error)
}
}
func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
markerPath := filepath.Join(t.TempDir(), "retry-backoff-marker")
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry Backoff 项目", Description: "retry backoff"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "重试延迟工作流",
Description: "retry backoff",
Type: domain.WorkflowTypeBatch,
MaxRetryAttempts: 1,
RetryBackoffSeconds: 2,
Nodes: []domain.WorkflowNode{
{
ID: "flaky-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "UTIL",
Description: "flaky",
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
Hyperparams: []domain.HyperParam{{
Key: "COMMAND",
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
}},
},
Position: domain.Position{X: 0, Y: 0},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create initial run: %v", err)
}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("create runtime: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("first process once: %v", err)
}
runs, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 2 {
t.Fatalf("expected original + retry run, got %d", len(runs))
}
var retryRun domain.RunRecord
for _, candidate := range runs {
if candidate.ID != initialRun.ID {
retryRun = candidate
break
}
}
if retryRun.AvailableAt == "" {
t.Fatalf("expected retry run to carry availableAt")
}
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available before backoff expires: %v", err)
}
stillQueued, err := dataStore.GetRun(ctx, retryRun.ID)
if err != nil {
t.Fatalf("get retry run before backoff: %v", err)
}
if stillQueued == nil || stillQueued.Status != domain.RunStatusQueued {
t.Fatalf("expected retry run to remain queued before backoff expires, got %#v", stillQueued)
}
time.Sleep(2200 * time.Millisecond)
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available after backoff expires: %v", err)
}
completedRetry := waitForRunStatus(t, ctx, dataStore, retryRun.ID, domain.RunStatusCompleted, 8*time.Second)
if completedRetry.Attempt != 2 {
t.Fatalf("expected retry attempt number to stay 2, got %d", completedRetry.Attempt)
}
}
func newReconcileFixture(t *testing.T, dataStore store.Store, name string) (*domain.WorkflowRecord, *domain.RunRecord) {
t.Helper()
ctx := context.Background()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: name + " project", Description: "reconcile"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: name + " workflow",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "node-1", Type: "operator", Data: domain.WorkflowNodeData{Label: name + " node"},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
startedAt := time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano)
run.Status = domain.RunStatusRunning
run.CurrentNodeID = "node-1"
run.StartedAt = startedAt
run.UpdatedAt = startedAt
run.NodeExecutions = []domain.NodeExecutionRecord{{
NodeID: "node-1", NodeLabel: name + " node", Attempt: 1, Status: domain.NodeExecutionStatusRunning, StartedAt: startedAt,
}}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
return workflow, run
}
func TestRuntimeReconcileDefersToAnUnexpiredExecutionLease(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, run := newReconcileFixture(t, dataStore, "leased execution")
run.ExecutionLeaseOwner = "worker-with-active-launch"
run.ExecutionLeaseExpiresAt = time.Now().UTC().Add(10 * time.Minute).Format(time.RFC3339Nano)
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert leased run: %v", err)
}
inspectCalls := 0
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) {
inspectCalls++
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
}})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile leased run: %v", err)
}
stored, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get leased run: %v", err)
}
if inspectCalls != 0 {
t.Fatalf("expected active lease to suppress concurrent Job inspection, got %d calls", inspectCalls)
}
if stored.Status != domain.RunStatusRunning || mustNodeExecution(t, stored, "node-1").Status != domain.NodeExecutionStatusRunning {
t.Fatalf("expected leased execution to remain untouched, got %#v", stored)
}
}
func TestRuntimeReconcileFailedJobTerminalizesRunAndNode(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
workflow, run := newReconcileFixture(t, dataStore, "failed job")
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) {
return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "container exited with code 2"}, nil
}})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile active runs: %v", err)
}
failed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "code 2") {
t.Fatalf("expected failed run with diagnostics, got %#v", failed)
}
if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_failed" || failed.Failure.NodeID != "node-1" {
t.Fatalf("expected structured kubernetes failure, got %#v", failed.Failure)
}
if execution := mustNodeExecution(t, failed, "node-1"); execution.Status != domain.NodeExecutionStatusFailed {
t.Fatalf("expected failed node execution, got %#v", execution)
}
storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
if err != nil || storedWorkflow.Status != domain.WorkflowStatusFailed {
t.Fatalf("expected failed workflow, got %#v err=%v", storedWorkflow, err)
}
}
func TestRuntimeReconcileSucceededJobFailsOrphanRunWhenWorkflowWasDeleted(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, run := newReconcileFixture(t, dataStore, "deleted workflow")
run.WorkflowID = "workflow-deleted-during-run"
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("orphan active run: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) {
return &k8s.JobInspection{State: k8s.JobStateSucceeded}, nil
}})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile orphan run: %v", err)
}
failed, err := dataStore.GetRun(ctx, run.ID)
if err != nil || failed == nil {
t.Fatalf("get orphan run: run=%#v err=%v", failed, err)
}
if failed.Status != domain.RunStatusFailed || failed.Failure == nil || failed.Failure.Code != "workflow_not_found" {
t.Fatalf("expected orphan run to fail terminally, got %#v", failed)
}
}
func TestRuntimeReconcileStuckImagePullTerminalizesButHealthyLongJobStaysRunning(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, blockedRun := newReconcileFixture(t, dataStore, "blocked image")
_, healthyRun := newReconcileFixture(t, dataStore, "healthy training")
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) {
if runID == blockedRun.ID {
return &k8s.JobInspection{
State: k8s.JobStatePending, CreatedAt: time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano),
BlockingReason: "ImagePullBackOff", BlockingMessage: "back-off pulling image",
}, nil
}
return &k8s.JobInspection{State: k8s.JobStateActive, CreatedAt: time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339Nano)}, nil
}})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("reconcile active runs: %v", err)
}
blocked, _ := dataStore.GetRun(ctx, blockedRun.ID)
healthy, _ := dataStore.GetRun(ctx, healthyRun.ID)
if blocked.Status != domain.RunStatusFailed || !strings.Contains(blocked.Error, "ImagePullBackOff") {
t.Fatalf("expected blocked image run to fail explicitly, got %#v", blocked)
}
if blocked.Failure == nil || blocked.Failure.Code != "kubernetes_image_pull_blocked" || blocked.Failure.Stage != "scheduling" {
t.Fatalf("expected structured image pull failure, got %#v", blocked.Failure)
}
if healthy.Status != domain.RunStatusRunning {
t.Fatalf("expected healthy long-running job unchanged, got %#v", healthy)
}
}
func TestRuntimeReconcileMissingJobFailsAfterRecoveryLimit(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, run := newReconcileFixture(t, dataStore, "missing job")
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
runtime.missingMaxRecoveries = 1
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("first reconcile: %v", err)
}
recovered, _ := dataStore.GetRun(ctx, run.ID)
if recovered.Status != domain.RunStatusQueued || !strings.Contains(mustNodeExecution(t, recovered, "node-1").Error, "recovery attempt 1/1") {
t.Fatalf("expected first recovery to requeue, got %#v", recovered)
}
recovered.Status = domain.RunStatusRunning
recovered.CurrentNodeID = "node-1"
recovered.UpdatedAt = time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano)
execution := mustNodeExecution(t, recovered, "node-1")
execution.Status = domain.NodeExecutionStatusRunning
execution.StartedAt = recovered.UpdatedAt
recovered.NodeExecutions = []domain.NodeExecutionRecord{execution}
if err := dataStore.UpsertRun(ctx, *recovered); err != nil {
t.Fatalf("simulate recovered execution: %v", err)
}
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("second reconcile: %v", err)
}
failed, _ := dataStore.GetRun(ctx, run.ID)
if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "after 1 recovery attempts") {
t.Fatalf("expected missing job to fail after recovery limit, got %#v", failed)
}
if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_missing" {
t.Fatalf("expected structured missing-job failure, got %#v", failed.Failure)
}
}
func TestRuntimeLaunchMissingJobRequeuesAndUsesNextNodeAttempt(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "missing launch", Description: "recover active launch"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "missing launch workflow",
Description: "recover active launch",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "train", Type: "operator", Data: domain.WorkflowNodeData{Label: "train"},
}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
launchAttempts := make([]int, 0, 2)
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(_ context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
launchAttempts = append(launchAttempts, input.Attempt)
if len(launchAttempts) == 1 {
return nil, &k8s.JobNotFoundError{JobName: input.RunID + "-" + input.NodeID + "-a1"}
}
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
return nil, err
}
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: input.RunID + "-" + input.NodeID + "-a2", Namespace: "droboticflow"},
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, nil
}})
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("first process once: %v", err)
}
requeued, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get requeued run: %v", err)
}
if requeued.Status != domain.RunStatusQueued || requeued.Failure != nil {
t.Fatalf("expected missing Job to be requeued without terminal failure, got %#v", requeued)
}
firstExecution := mustNodeExecution(t, requeued, "train")
if firstExecution.Status != domain.NodeExecutionStatusFailed || !strings.Contains(firstExecution.Error, "recovery attempt 1/3") {
t.Fatalf("expected bounded missing-Job recovery evidence, got %#v", firstExecution)
}
requeued.AvailableAt = nowISO()
if err := dataStore.UpsertRun(ctx, *requeued); err != nil {
t.Fatalf("make recovered run available: %v", err)
}
if _, err := runtime.ProcessOnce(ctx); err != nil {
t.Fatalf("second process once: %v", err)
}
completed, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get completed run: %v", err)
}
if completed.Status != domain.RunStatusCompleted {
t.Fatalf("expected recovered run to complete, got %#v", completed)
}
if len(launchAttempts) != 2 || launchAttempts[0] != 1 || launchAttempts[1] != 2 {
t.Fatalf("expected Kubernetes attempts [1 2], got %#v", launchAttempts)
}
if len(completed.K8sRefs) != 1 || completed.K8sRefs[0].Attempt != 2 || !strings.HasSuffix(completed.K8sRefs[0].JobName, "-a2") {
t.Fatalf("expected recovered Job reference to retain attempt 2, got %#v", completed.K8sRefs)
}
}
func TestRuntimeReconcileIsolatesPerRunInspectionErrors(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, brokenRun := newReconcileFixture(t, dataStore, "inspection error")
_, failedRun := newReconcileFixture(t, dataStore, "independent failure")
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) {
if runID == brokenRun.ID {
return nil, fmt.Errorf("temporary kubernetes error")
}
return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "independent failure"}, nil
}})
if err := runtime.reconcileActiveRuns(ctx); err != nil {
t.Fatalf("per-run error must not fail the cycle: %v", err)
}
broken, _ := dataStore.GetRun(ctx, brokenRun.ID)
failed, _ := dataStore.GetRun(ctx, failedRun.ID)
if broken.Status != domain.RunStatusRunning {
t.Fatalf("expected errored inspection unchanged, got %#v", broken)
}
if failed.Status != domain.RunStatusFailed {
t.Fatalf("expected independent run reconciled, got %#v", failed)
}
}
func TestRuntimeLeaseOwnerChangeCancelsExecutionContext(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, run := newReconcileFixture(t, dataStore, "lease loss")
run.ExecutionLeaseOwner = "other-worker"
run.ExecutionLeaseExpiresAt = time.Now().UTC().Add(time.Hour).Format(time.RFC3339Nano)
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert foreign lease: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
runtime.dispatchOwner = "this-worker"
runtime.runLeaseTTL = 30 * time.Millisecond
leaseCtx, cancelLease := context.WithCancelCause(ctx)
defer cancelLease(context.Canceled)
go runtime.renewRunLeaseUntil(leaseCtx, run.ID, cancelLease)
deadline := time.After(time.Second)
for !errors.Is(context.Cause(leaseCtx), errRunLeaseLost) {
select {
case <-deadline:
t.Fatalf("expected lease owner change to cancel execution context, cause=%v", context.Cause(leaseCtx))
case <-time.After(5 * time.Millisecond):
}
}
}
func TestRuntimeProcessAvailableAppliesPersistedPollInterval(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
settings.Worker.PollIntervalMS = 750
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("upsert settings: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
runtime.SetConfiguredPollInterval(2 * time.Second)
if err := runtime.ProcessAvailable(ctx); err != nil {
t.Fatalf("process available: %v", err)
}
if got := runtime.EffectivePollInterval(); got != 750*time.Millisecond {
t.Fatalf("expected persisted 750ms poll interval, got %s", got)
}
}
func TestLatestStructuredProgressUsesLatestValidMarker(t *testing.T) {
logs := strings.Join([]string{
"ordinary training output",
`DROBOTICFLOW_PROGRESS {"stage":"train","epoch":1,"step":20,"maxStep":100,"samplesPerSecond":4.5,"estimatedRemainingSeconds":18,"checkpoint":"checkpoint-20","updatedAt":"2026-07-10T08:00:00Z"}`,
"DROBOTICFLOW_PROGRESS {partial",
`prefix DROBOTICFLOW_PROGRESS {"stage":"train","epoch":2,"step":40,"maxStep":100,"samplesPerSecond":5.5,"estimatedRemainingSeconds":11,"checkpoint":"checkpoint-40","updatedAt":"2026-07-10T08:01:00Z"}`,
}, "\n")
progress, err := latestStructuredProgress(logs)
if err != nil {
t.Fatalf("parse progress: %v", err)
}
if progress == nil || progress.Epoch != 2 || progress.Step != 40 || progress.Checkpoint != "checkpoint-40" {
t.Fatalf("expected latest valid marker, got %#v", progress)
}
}
func TestNodeProgressReporterPersistsStructuredMarker(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
_, run := newReconcileFixture(t, dataStore, "structured progress")
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{tailNodePodLogs: func(context.Context, string, string, int64) (string, error) {
return `DROBOTICFLOW_PROGRESS {"stage":"train","epoch":3,"step":75,"maxStep":100,"samplesPerSecond":6.25,"estimatedRemainingSeconds":4,"checkpoint":"checkpoint-75"}`, nil
}})
reporterCtx, cancelReporter := context.WithCancel(ctx)
done := make(chan struct{})
go runtime.runNodeProgressReporterWithInterval(reporterCtx, run.ID, "node-1", false, 5*time.Millisecond, func() { close(done) })
deadline := time.Now().Add(time.Second)
for {
stored, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if stored.Progress != nil {
if stored.Progress.Step != 75 || stored.Progress.Checkpoint != "checkpoint-75" || stored.Progress.UpdatedAt == "" {
t.Fatalf("unexpected structured progress: %#v", stored.Progress)
}
break
}
if time.Now().After(deadline) {
t.Fatal("timed out waiting for structured progress")
}
time.Sleep(5 * time.Millisecond)
}
cancelReporter()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("progress reporter did not stop")
}
}
func TestSyncStopMetadataPreservesAtomicProgressUpdates(t *testing.T) {
target := &domain.RunRecord{}
source := &domain.RunRecord{Progress: &domain.RunProgress{Stage: "train", Step: 9, UpdatedAt: "2026-07-10T08:00:00Z"}}
syncStopMetadata(target, source)
if target.Progress == nil || target.Progress.Step != 9 {
t.Fatalf("expected progress copied from persisted run, got %#v", target.Progress)
}
source.Progress.Step = 10
if target.Progress.Step != 9 {
t.Fatalf("expected progress copy to avoid pointer aliasing, got %#v", target.Progress)
}
}
func TestCompletedNodePersistsTerminalStructuredProgressMarker(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "terminal progress", Description: "progress"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID, Name: "terminal progress", Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{ID: "train", Type: "operator", Data: domain.WorkflowNodeData{Label: "train"}}},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create run: %v", err)
}
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(_ context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
stageManifestPath := input.Env["STAGE_MANIFEST_PATH"]
if err := os.MkdirAll(filepath.Dir(stageManifestPath), 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(stageManifestPath, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
return nil, err
}
return &k8s.LaunchResult{
Reference: &k8s.JobReference{JobName: "terminal-progress", Namespace: "droboticflow"},
Output: strings.Join([]string{
`DROBOTICFLOW_PROGRESS {"stage":"completed","epoch":4,"step":100,"maxStep":100,"samplesPerSecond":6.5,"estimatedRemainingSeconds":0,"checkpoint":"checkpoint-final"}`,
`DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
}, "\n"),
}, nil
}})
result, err := runtime.ProcessOnce(ctx)
if err != nil {
t.Fatalf("process run: %v", err)
}
if result == nil || result.Run == nil || result.Run.ID != run.ID {
t.Fatalf("unexpected process result: %#v", result)
}
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, time.Second)
if completed.Progress == nil || completed.Progress.Stage != "completed" || completed.Progress.Step != 100 || completed.Progress.Checkpoint != "checkpoint-final" {
t.Fatalf("expected terminal structured progress, got %#v", completed.Progress)
}
}
// shouldReportNodeProgress must flag only custom-container operators — builtin
// operator-runner pods self-report resourceSeries via their in-process ResourceSampler,
// so platform-side GPU sampling is reserved for nodes whose resolved image differs from
// the operator-runner image (mirroring k8s.Launch's shouldSample gate).
func TestShouldReportNodeProgressOnlyForCustomContainers(t *testing.T) {
rt := &Runtime{operatorRunnerImage: "droboticflow-operator-runner:latest"}
cases := []struct {
name string
node domain.WorkflowNode
want bool
}{
{"custom image", domain.WorkflowNode{Data: domain.WorkflowNodeData{Image: "sam3-video-tracker:latest"}}, true},
{"builtin operator-runner image", domain.WorkflowNode{Data: domain.WorkflowNodeData{Image: "droboticflow-operator-runner:latest"}}, false},
{"digest-pinned custom image", domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorImageID: "sam3-video-tracker@sha256:40efc6ba77e0bfb3a029825b20a36615445f7c62a0279804971c543c46ccd08f"}}, true},
{"no image falls back to operator-runner", domain.WorkflowNode{Data: domain.WorkflowNodeData{}}, false},
}
for _, tc := range cases {
if got := shouldReportNodeProgress(rt, tc.node); got != tc.want {
t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
}
}
}
// mapGPUSample must round-trip every GPU-relevant field the launcher reports onto the
// resource monitor's domain.ResourceSample (the same mapping the executor uses when
// folding the launcher's final series).
func TestMapGPUSampleCopiesFields(t *testing.T) {
in := k8s.GPUSample{
Timestamp: "2026-07-10T00:00:00.000000Z",
GPU: 42,
GPUMemoryUsedMiB: 28000,
GPUMemoryTotalMiB: 81920,
GPUName: "NVIDIA A800-SXM4-80GB",
GPUCount: 1,
}
out := mapGPUSample(in)
if out.Timestamp != in.Timestamp || out.GPU != in.GPU || out.GPUMemoryUsedMiB != in.GPUMemoryUsedMiB ||
out.GPUMemoryTotalMiB != in.GPUMemoryTotalMiB || out.GPUName != in.GPUName || out.GPUCount != in.GPUCount {
t.Fatalf("mapGPUSample mismatch: in=%#v out=%#v", in, out)
}
}