4976 lines
177 KiB
Go
4976 lines
177 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
type testLauncher struct {
|
|
launch func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error)
|
|
inspect func(context.Context, string, string) (*k8s.JobInspection, error)
|
|
gpuCapacity func(context.Context) (*k8s.GPUCapacitySnapshot, error)
|
|
terminateRun func(context.Context, string, string) error
|
|
samplePodGPUForRun func(context.Context, string, string) (k8s.GPUSample, error)
|
|
tailNodePodLogs func(context.Context, string, string, int64) (string, error)
|
|
}
|
|
|
|
func (launcher testLauncher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
|
|
if launcher.tailNodePodLogs == nil {
|
|
return "", fmt.Errorf("no pod logs for run %s node %s", runID, nodeID)
|
|
}
|
|
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines)
|
|
}
|
|
|
|
func (launcher testLauncher) Launch(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
if launcher.launch == nil {
|
|
return nil, fmt.Errorf("test launcher launch not configured")
|
|
}
|
|
return launcher.launch(ctx, input)
|
|
}
|
|
|
|
func (launcher testLauncher) Inspect(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) {
|
|
if launcher.inspect == nil {
|
|
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
|
|
}
|
|
return launcher.inspect(ctx, runID, nodeID)
|
|
}
|
|
|
|
func (launcher testLauncher) TerminateRun(ctx context.Context, namespace string, runID string) error {
|
|
if launcher.terminateRun == nil {
|
|
return nil
|
|
}
|
|
return launcher.terminateRun(ctx, namespace, runID)
|
|
}
|
|
|
|
func (launcher testLauncher) SamplePodGPUForRun(ctx context.Context, runID string, nodeID string) (k8s.GPUSample, error) {
|
|
if launcher.samplePodGPUForRun == nil {
|
|
return k8s.GPUSample{}, fmt.Errorf("no running pod for run %s node %s", runID, nodeID)
|
|
}
|
|
return launcher.samplePodGPUForRun(ctx, runID, nodeID)
|
|
}
|
|
|
|
func (launcher testLauncher) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) {
|
|
if launcher.gpuCapacity == nil {
|
|
return &k8s.GPUCapacitySnapshot{ResourceName: "nvidia.com/gpu", SelectorSummary: "<none>"}, nil
|
|
}
|
|
return launcher.gpuCapacity(ctx)
|
|
}
|
|
|
|
func operatorRunnerPath(t *testing.T) string {
|
|
t.Helper()
|
|
path, err := filepath.Abs("../../backend/operator-runner/main.py")
|
|
if err != nil {
|
|
t.Fatalf("resolve operator runner: %v", err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func newTestRuntime(t *testing.T, dataStore store.Store) (*Runtime, error) {
|
|
t.Helper()
|
|
t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "true")
|
|
runnerPath := operatorRunnerPath(t)
|
|
return &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "operator-runner:latest",
|
|
workspaceRoot: t.TempDir(),
|
|
launcher: testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
env := append([]string{}, os.Environ()...)
|
|
keys := make([]string, 0, len(input.Env))
|
|
for key := range input.Env {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
env = append(env,
|
|
"RUN_ID="+input.RunID,
|
|
"WORKFLOW_ID="+input.WorkflowID,
|
|
"NODE_ID="+input.NodeID,
|
|
"NODE_LABEL="+input.NodeLabel,
|
|
)
|
|
for _, key := range keys {
|
|
env = append(env, key+"="+input.Env[key])
|
|
}
|
|
cmd := exec.CommandContext(ctx, "python3", runnerPath)
|
|
cmd.Env = env
|
|
output, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: %s", err, string(output))
|
|
}
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{
|
|
JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID),
|
|
Namespace: "droboticflow",
|
|
},
|
|
Output: string(output),
|
|
}, nil
|
|
},
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func newTestRuntimeWithLauncher(t *testing.T, dataStore store.Store, launcher jobLauncher) *Runtime {
|
|
t.Helper()
|
|
t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "true")
|
|
return &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "operator-runner:latest",
|
|
workspaceRoot: t.TempDir(),
|
|
launcher: launcher,
|
|
}
|
|
}
|
|
|
|
func mustNodeExecution(t *testing.T, run *domain.RunRecord, nodeID string) domain.NodeExecutionRecord {
|
|
t.Helper()
|
|
for _, item := range run.NodeExecutions {
|
|
if item.NodeID == nodeID {
|
|
return item
|
|
}
|
|
}
|
|
t.Fatalf("missing node execution for %s", nodeID)
|
|
return domain.NodeExecutionRecord{}
|
|
}
|
|
|
|
func mustArtifact(t *testing.T, execution domain.NodeExecutionRecord, name string) domain.ExecutionArtifact {
|
|
t.Helper()
|
|
for _, item := range execution.Artifacts {
|
|
if item.Name == name {
|
|
return item
|
|
}
|
|
}
|
|
t.Fatalf("missing artifact %s on node %s", name, execution.NodeID)
|
|
return domain.ExecutionArtifact{}
|
|
}
|
|
|
|
func mustArtifactPayload(t *testing.T, artifact domain.ExecutionArtifact) map[string]any {
|
|
t.Helper()
|
|
if artifact.Data != nil {
|
|
if dataRef, ok := artifact.Data["dataRef"].(map[string]any); ok {
|
|
if payloadPath, ok := dataRef["payloadPath"].(string); ok && strings.TrimSpace(payloadPath) != "" {
|
|
payload, err := os.ReadFile(payloadPath)
|
|
if err != nil {
|
|
t.Fatalf("read artifact payload %s: %v", payloadPath, err)
|
|
}
|
|
var decoded map[string]any
|
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
|
t.Fatalf("decode artifact payload %s: %v", payloadPath, err)
|
|
}
|
|
return decoded
|
|
}
|
|
}
|
|
return artifact.Data
|
|
}
|
|
if strings.TrimSpace(artifact.Path) != "" {
|
|
payload, err := os.ReadFile(artifact.Path)
|
|
if err == nil {
|
|
var decoded map[string]any
|
|
if err := json.Unmarshal(payload, &decoded); err == nil {
|
|
return decoded
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("artifact %s has no structured payload", artifact.Name)
|
|
return nil
|
|
}
|
|
|
|
func mustProcessOnceRun(t *testing.T, runtime *Runtime, ctx context.Context) *domain.RunRecord {
|
|
t.Helper()
|
|
result, err := runtime.ProcessOnce(ctx)
|
|
if err != nil {
|
|
t.Fatalf("process run: %v", err)
|
|
}
|
|
if result == nil || result.Run == nil {
|
|
t.Fatalf("expected process-once to return a run, got %#v", result)
|
|
}
|
|
return result.Run
|
|
}
|
|
|
|
func TestProcessOnceResultJSONUsesLightweightRunSummary(t *testing.T) {
|
|
run := &domain.RunRecord{
|
|
ID: "run-1",
|
|
WorkflowID: "workflow-1",
|
|
WorkflowName: "workflow",
|
|
ProjectID: "project-1",
|
|
ProjectName: "project",
|
|
Status: domain.RunStatusCompleted,
|
|
Attempt: 2,
|
|
CurrentNodeID: "node-2",
|
|
ExecutionLeaseOwner: "worker-1",
|
|
ExecutionLeaseExpiresAt: "2026-07-02T00:00:00Z",
|
|
UpdatedAt: "2026-07-02T00:00:01Z",
|
|
Logs: []domain.LogEntryRecord{{
|
|
Level: "INFO",
|
|
Message: strings.Repeat("log-", 100),
|
|
}},
|
|
NodeExecutions: []domain.NodeExecutionRecord{{
|
|
NodeID: "node-1",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
ID: "artifact-1",
|
|
Name: "large.json",
|
|
Data: map[string]any{"payload": strings.Repeat("data-", 100)},
|
|
}},
|
|
Logs: []domain.LogEntryRecord{{Message: strings.Repeat("node-log-", 100)}},
|
|
}},
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
ID: "run-artifact-1",
|
|
Name: "report.json",
|
|
Data: map[string]any{"payload": strings.Repeat("report-", 100)},
|
|
}},
|
|
}
|
|
result := newProcessOnceResult(ProcessOnceStatusClaimed, run)
|
|
if result.Run != run {
|
|
t.Fatalf("expected runtime result to retain the full run pointer")
|
|
}
|
|
encoded, err := json.Marshal(result)
|
|
if err != nil {
|
|
t.Fatalf("marshal process-once result: %v", err)
|
|
}
|
|
body := string(encoded)
|
|
for _, forbidden := range []string{"nodeExecutions", "logs", "resourceSeries", "k8sRefs", "large.json", "node-log-", "report-"} {
|
|
if strings.Contains(body, forbidden) {
|
|
t.Fatalf("process-once response leaked full run field %q in %s", forbidden, body)
|
|
}
|
|
}
|
|
for _, expected := range []string{`"status":"claimed"`, `"id":"run-1"`, `"nodeExecutionCount":1`, `"artifactCount":2`, `"currentNodeId":"node-2"`} {
|
|
if !strings.Contains(body, expected) {
|
|
t.Fatalf("process-once response missing %s in %s", expected, body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableSkipsAlreadyActiveRun(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "active run", Description: "skip"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "workflow",
|
|
Description: "skip active run",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "source",
|
|
Outputs: []domain.PortDefinition{{
|
|
ID: "out-1",
|
|
Name: "payload",
|
|
Type: domain.PortTypeJSON,
|
|
}},
|
|
},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
launchCount := 0
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
launchCount++
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: "job-" + input.RunID + "-" + input.NodeID, Namespace: "droboticflow"},
|
|
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
|
}, nil
|
|
},
|
|
})
|
|
if !runtime.tryMarkRunActive(run.ID) {
|
|
t.Fatal("expected active run mark to succeed")
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
if launchCount != 0 {
|
|
t.Fatalf("expected active run to be skipped, got %d launches", launchCount)
|
|
}
|
|
}
|
|
|
|
// TestProcessOnceSkippedLeaseActiveReleasesLeaseAndKeepsRunning guards the race
|
|
// where ProcessOnce's ClaimNextQueuedRun wins the mongo claim but tryMarkRunActive
|
|
// fails because another goroutine in this worker already owns the run.
|
|
//
|
|
// The old behavior reverted status to Queued while leaving currentNodeId set and
|
|
// leaving the just-claimed lease held — an inconsistent "queued+currentNodeId"
|
|
// state no other queued-writer emits (and the one that matched a phantom symptom
|
|
// we chased). It also blocked ProcessAvailable's claim filter for the lease TTL
|
|
// (~15min), orphaning the run. The fix releases the spurious lease and leaves
|
|
// status=Running (the active processor owns the run).
|
|
func TestProcessOnceSkippedLeaseActiveReleasesLeaseAndKeepsRunning(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "lease race", Description: "skip"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "workflow",
|
|
Description: "lease race",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{Label: "source"},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
launchCount := 0
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
launchCount++
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: "job-" + input.RunID + "-" + input.NodeID, Namespace: "droboticflow"},
|
|
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
|
}, nil
|
|
},
|
|
})
|
|
|
|
// Pre-mark the run active so ProcessOnce's tryMarkRunActive fails, simulating
|
|
// the race where another goroutine already owns the in-memory activeRuns slot.
|
|
if !runtime.tryMarkRunActive(run.ID) {
|
|
t.Fatal("expected active run mark to succeed")
|
|
}
|
|
|
|
result, err := runtime.ProcessOnce(ctx)
|
|
if err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
if result == nil || result.Status != ProcessOnceStatusSkippedLeaseActive {
|
|
t.Fatalf("expected skipped_lease_active, got %#v", result)
|
|
}
|
|
if launchCount != 0 {
|
|
t.Fatalf("expected no launch (run already active), got %d", launchCount)
|
|
}
|
|
|
|
stored, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if stored.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected run to stay Running after skipped claim (active processor owns it), got %s", stored.Status)
|
|
}
|
|
if strings.TrimSpace(stored.ExecutionLeaseOwner) != "" {
|
|
t.Fatalf("expected spurious lease released so it does not block the next claim for the lease TTL, got owner=%q", stored.ExecutionLeaseOwner)
|
|
}
|
|
}
|
|
|
|
func TestExecuteNodeFallsBackToOperatorResourceDefaults(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
operator, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
|
|
Name: "OpenVLA 微调训练",
|
|
Type: "builtin",
|
|
Category: "AI",
|
|
Description: "gpu operator",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "model", Type: domain.PortTypeFile}},
|
|
Resources: &domain.OperatorResources{
|
|
CPURequest: "8",
|
|
CPULimit: "24",
|
|
MemoryRequest: "32Gi",
|
|
MemoryLimit: "128Gi",
|
|
GPULimit: "1",
|
|
SharedMemorySize: "12Gi",
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create operator: %v", err)
|
|
}
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "GPU 项目", Description: "resources"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "GPU 工作流",
|
|
Description: "operator resource defaults",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "train",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "OpenVLA 微调训练",
|
|
OperatorID: operator.ID,
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "model", Type: domain.PortTypeFile}},
|
|
},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
var captured k8s.LaunchInput
|
|
launchCount := 0
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
launchCount++
|
|
captured = input
|
|
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
|
|
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o600); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: "job-" + input.NodeID, Namespace: "droboticflow"},
|
|
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
|
}, nil
|
|
},
|
|
})
|
|
if _, _, err := runtime.executeNode(ctx, run, workflow.Nodes[0], nil); err != nil {
|
|
t.Fatalf("execute node: %v", err)
|
|
}
|
|
if launchCount != 1 {
|
|
t.Fatalf("expected one launch, got %d", launchCount)
|
|
}
|
|
if captured.GPULimit != "1" || captured.Env["GPU_LIMIT"] != "1" {
|
|
t.Fatalf("expected operator GPU default to reach launcher/env, got gpu=%q env=%q", captured.GPULimit, captured.Env["GPU_LIMIT"])
|
|
}
|
|
if captured.CPURequest != "8" || captured.CPULimit != "24" || captured.MemoryRequest != "32Gi" || captured.MemoryLimit != "128Gi" {
|
|
t.Fatalf("expected operator resource defaults, got %#v", captured)
|
|
}
|
|
if captured.SharedMemorySize != "12Gi" {
|
|
t.Fatalf("expected operator shared memory default, got %q", captured.SharedMemorySize)
|
|
}
|
|
}
|
|
|
|
func TestResolveEffectiveNodeSharedMemoryPrecedence(t *testing.T) {
|
|
operatorResources := &domain.OperatorResources{SharedMemorySize: "8Gi"}
|
|
operatorDefault := resolveEffectiveNodeResources(domain.WorkflowNode{}, nil, operatorResources)
|
|
if operatorDefault.SharedMemorySize != "8Gi" {
|
|
t.Fatalf("operator shared memory default = %q, want 8Gi", operatorDefault.SharedMemorySize)
|
|
}
|
|
|
|
nodeOverride := resolveEffectiveNodeResources(domain.WorkflowNode{
|
|
Data: domain.WorkflowNodeData{SharedMemorySize: "16Gi"},
|
|
}, nil, operatorResources)
|
|
if nodeOverride.SharedMemorySize != "16Gi" {
|
|
t.Fatalf("node shared memory override = %q, want 16Gi", nodeOverride.SharedMemorySize)
|
|
}
|
|
|
|
platformFallback := resolveEffectiveNodeResources(domain.WorkflowNode{}, nil, nil)
|
|
if platformFallback.SharedMemorySize != "" {
|
|
t.Fatalf("empty shared memory should be left for launcher platform fallback, got %q", platformFallback.SharedMemorySize)
|
|
}
|
|
}
|
|
|
|
func TestProcessRunFailsInvalidWorkflowWithDuplicateInputEdges(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "非法工作流项目",
|
|
Description: "duplicate input edges",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "非法工作流",
|
|
Description: "duplicate input edges",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source-a",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "上游 A",
|
|
Category: "IO",
|
|
Description: "source a",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
|
|
},
|
|
},
|
|
{
|
|
ID: "source-b",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "上游 B",
|
|
Category: "IO",
|
|
Description: "source b",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
|
|
},
|
|
},
|
|
{
|
|
ID: "upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "upload",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "result", Type: domain.PortTypeJSON}},
|
|
},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "edge-a", Source: "source-a", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
|
|
{ID: "edge-b", Source: "source-b", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
processed, err := runtime.processRun(ctx, run)
|
|
if err != nil {
|
|
t.Fatalf("process run: %v", err)
|
|
}
|
|
if processed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %s", processed.Status)
|
|
}
|
|
if !strings.Contains(processed.Error, "同一输入端口存在多条上游连线") {
|
|
t.Fatalf("expected duplicate input edge error, got %q", processed.Error)
|
|
}
|
|
if len(processed.NodeExecutions) != 0 {
|
|
t.Fatalf("expected no node executions for invalid workflow, got %#v", processed.NodeExecutions)
|
|
}
|
|
}
|
|
|
|
func TestProcessRunFailsInvalidIncrementalDirectoryDepth(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "非法目录深度项目",
|
|
Description: "invalid incremental depth",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "非法目录深度工作流",
|
|
Description: "invalid incremental depth",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
|
|
{Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"},
|
|
{Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "zero"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
processed := mustProcessOnceRun(t, runtime, ctx)
|
|
if processed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %s", processed.Status)
|
|
}
|
|
if !strings.Contains(processed.Error, "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") {
|
|
t.Fatalf("expected directory depth error, got %q", processed.Error)
|
|
}
|
|
execution := mustNodeExecution(t, processed, "source")
|
|
if execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed source execution, got %s", execution.Status)
|
|
}
|
|
if !strings.Contains(execution.Error, "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer") {
|
|
t.Fatalf("expected node directory depth error, got %q", execution.Error)
|
|
}
|
|
}
|
|
|
|
func TestProcessRunFailsInvalidIncrementalUnit(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "非法增量单位项目",
|
|
Description: "invalid incremental unit",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "非法增量单位工作流",
|
|
Description: "invalid incremental unit",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
|
|
{Key: "INCREMENTAL_UNIT", Value: "ROW"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
processed := mustProcessOnceRun(t, runtime, ctx)
|
|
if processed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %s", processed.Status)
|
|
}
|
|
if !strings.Contains(processed.Error, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY") {
|
|
t.Fatalf("expected incremental unit error, got %q", processed.Error)
|
|
}
|
|
execution := mustNodeExecution(t, processed, "source")
|
|
if execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed source execution, got %s", execution.Status)
|
|
}
|
|
if !strings.Contains(execution.Error, "INCREMENTAL_UNIT must be OBJECT or DIRECTORY") {
|
|
t.Fatalf("expected node incremental unit error, got %q", execution.Error)
|
|
}
|
|
}
|
|
|
|
func TestProcessRunFailsInvalidScanMode(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "非法扫描模式项目",
|
|
Description: "invalid scan mode",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "非法扫描模式工作流",
|
|
Description: "invalid scan mode",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "ROW"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
processed := mustProcessOnceRun(t, runtime, ctx)
|
|
if processed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %s", processed.Status)
|
|
}
|
|
if !strings.Contains(processed.Error, "SCAN_MODE must be FULL or INCREMENTAL") {
|
|
t.Fatalf("expected scan mode error, got %q", processed.Error)
|
|
}
|
|
execution := mustNodeExecution(t, processed, "source")
|
|
if execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed source execution, got %s", execution.Status)
|
|
}
|
|
if !strings.Contains(execution.Error, "SCAN_MODE must be FULL or INCREMENTAL") {
|
|
t.Fatalf("expected node scan mode error, got %q", execution.Error)
|
|
}
|
|
}
|
|
|
|
func TestProcessRunFailsInvalidDataSource(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "非法数据源项目",
|
|
Description: "invalid data source",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "非法数据源工作流",
|
|
Description: "invalid data source",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "ROW"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
processed := mustProcessOnceRun(t, runtime, ctx)
|
|
if processed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %s", processed.Status)
|
|
}
|
|
if !strings.Contains(processed.Error, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") {
|
|
t.Fatalf("expected data source error, got %q", processed.Error)
|
|
}
|
|
execution := mustNodeExecution(t, processed, "source")
|
|
if execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed source execution, got %s", execution.Status)
|
|
}
|
|
if !strings.Contains(execution.Error, "DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") {
|
|
t.Fatalf("expected node data source error, got %q", execution.Error)
|
|
}
|
|
}
|
|
|
|
func TestSourceStateFromCheckpointPrefersCheckpointFilePayload(t *testing.T) {
|
|
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
|
|
run := &domain.RunRecord{ID: "run-1"}
|
|
checkpointPath := filepath.Join(t.TempDir(), "source-checkpoint.json")
|
|
writeFile(t, checkpointPath, `{
|
|
"sourceFingerprint": "fp-1",
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test",
|
|
"bucket": "datastorage",
|
|
"prefix": "algo/X-platform-test",
|
|
"pattern": "",
|
|
"scanMode": "INCREMENTAL",
|
|
"directoryDepth": 1,
|
|
"seenFingerprints": ["a", "b"],
|
|
"listedCount": 28,
|
|
"newCount": 0,
|
|
"scanStartedAt": "2026-04-12T15:45:42Z",
|
|
"scanCompletedAt": "2026-04-12T15:45:55Z"
|
|
}`)
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: "scan-source",
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
Name: "source-checkpoint.json",
|
|
Path: checkpointPath,
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "fp-1",
|
|
"listedCount": 28.0,
|
|
"newCount": 28.0,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
state, err := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0])
|
|
if err != nil {
|
|
t.Fatalf("source state from checkpoint: %v", err)
|
|
}
|
|
if state == nil {
|
|
t.Fatalf("expected source state")
|
|
}
|
|
if state.LastNewCount != 0 {
|
|
t.Fatalf("expected checkpoint file payload to win, got lastNewCount=%d", state.LastNewCount)
|
|
}
|
|
if state.LastListedCount != 28 {
|
|
t.Fatalf("expected lastListedCount=28, got %d", state.LastListedCount)
|
|
}
|
|
if state.LastRunID != run.ID {
|
|
t.Fatalf("expected lastRunID=%s, got %s", run.ID, state.LastRunID)
|
|
}
|
|
}
|
|
|
|
func TestSourceStateFromCheckpointRejectsUnreadableOrInvalidCheckpointFiles(t *testing.T) {
|
|
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
|
|
run := &domain.RunRecord{ID: "run-1"}
|
|
execution := domain.NodeExecutionRecord{NodeID: "scan-source"}
|
|
|
|
missingPath := filepath.Join(t.TempDir(), "missing-source-checkpoint.json")
|
|
_, err := sourceStateFromCheckpoint(workflow, run, execution, &domain.ExecutionArtifact{
|
|
Name: "source-checkpoint.json",
|
|
Path: missingPath,
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "stale-inline",
|
|
"listedCount": 1,
|
|
"newCount": 1,
|
|
},
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "read source checkpoint") {
|
|
t.Fatalf("expected unreadable checkpoint error, got %v", err)
|
|
}
|
|
|
|
invalidPath := filepath.Join(t.TempDir(), "source-checkpoint.json")
|
|
writeFile(t, invalidPath, `{`)
|
|
_, err = sourceStateFromCheckpoint(workflow, run, execution, &domain.ExecutionArtifact{
|
|
Name: "source-checkpoint.json",
|
|
Path: invalidPath,
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "stale-inline",
|
|
"listedCount": 1,
|
|
"newCount": 1,
|
|
},
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "parse source checkpoint") {
|
|
t.Fatalf("expected invalid checkpoint JSON error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSourceStateFromCheckpointRejectsLossyCheckpointCounts(t *testing.T) {
|
|
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
|
|
run := &domain.RunRecord{ID: "run-1"}
|
|
execution := domain.NodeExecutionRecord{
|
|
NodeID: "scan-source",
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
Name: "source-checkpoint.json",
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "fp-lossy",
|
|
"directoryDepth": float64(1),
|
|
"listedCount": 28.5,
|
|
"newCount": 0.0,
|
|
},
|
|
}},
|
|
}
|
|
|
|
_, err := sourceStateFromCheckpoint(workflow, run, execution, &execution.Artifacts[0])
|
|
if err == nil || !strings.Contains(err.Error(), "listedCount must be an integer") {
|
|
t.Fatalf("expected lossy listedCount error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCommitSourceStatesPrefersPersistedRunSnapshot(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
workflow := &domain.WorkflowRecord{ID: "workflow-1"}
|
|
persistedRun := domain.RunRecord{
|
|
ID: "run-1",
|
|
WorkflowID: workflow.ID,
|
|
NodeExecutions: []domain.NodeExecutionRecord{{
|
|
NodeID: "scan-source",
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
Name: "source-checkpoint.json",
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "fp-1",
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test",
|
|
"bucket": "datastorage",
|
|
"prefix": "algo/X-platform-test",
|
|
"scanMode": "INCREMENTAL",
|
|
"directoryDepth": 1.0,
|
|
"seenFingerprints": []any{"a", "b"},
|
|
"listedCount": 28.0,
|
|
"newCount": 0.0,
|
|
"scanStartedAt": "2026-04-12T15:45:42Z",
|
|
"scanCompletedAt": "2026-04-12T15:45:55Z",
|
|
},
|
|
}},
|
|
}},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, persistedRun); err != nil {
|
|
t.Fatalf("upsert persisted run: %v", err)
|
|
}
|
|
inMemoryRun := &domain.RunRecord{
|
|
ID: "run-1",
|
|
WorkflowID: workflow.ID,
|
|
NodeExecutions: []domain.NodeExecutionRecord{{
|
|
NodeID: "scan-source",
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
Name: "source-checkpoint.json",
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "fp-1",
|
|
"directoryDepth": 1.0,
|
|
"listedCount": 28.0,
|
|
"newCount": 28.0,
|
|
},
|
|
}},
|
|
}},
|
|
}
|
|
|
|
if err := runtime.commitSourceStates(ctx, workflow, inMemoryRun); err != nil {
|
|
t.Fatalf("commit source states: %v", err)
|
|
}
|
|
state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-1")
|
|
if err != nil {
|
|
t.Fatalf("get source state: %v", err)
|
|
}
|
|
if state == nil {
|
|
t.Fatalf("expected committed source state")
|
|
}
|
|
if state.LastNewCount != 0 {
|
|
t.Fatalf("expected committed source state to use persisted run snapshot, got %d", state.LastNewCount)
|
|
}
|
|
}
|
|
|
|
func TestCommitSourceStatesPreservesSeenFingerprintsFromTypedSlice(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
workflow := &domain.WorkflowRecord{ID: "workflow-typed-slice"}
|
|
persistedRun := domain.RunRecord{
|
|
ID: "run-typed-slice",
|
|
WorkflowID: workflow.ID,
|
|
NodeExecutions: []domain.NodeExecutionRecord{{
|
|
NodeID: "scan-source",
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
Name: "source-checkpoint.json",
|
|
Data: map[string]any{
|
|
"sourceFingerprint": "fp-typed-slice",
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test/raw",
|
|
"bucket": "datastorage",
|
|
"prefix": "algo/X-platform-test/raw",
|
|
"scanMode": "INCREMENTAL",
|
|
"directoryDepth": 1.0,
|
|
"seenFingerprints": []string{"a", "b", "c"},
|
|
"listedCount": 4.0,
|
|
"newCount": 0.0,
|
|
},
|
|
}},
|
|
}},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, persistedRun); err != nil {
|
|
t.Fatalf("upsert persisted run: %v", err)
|
|
}
|
|
|
|
if err := runtime.commitSourceStates(ctx, workflow, &persistedRun); err != nil {
|
|
t.Fatalf("commit source states: %v", err)
|
|
}
|
|
state, err := dataStore.GetSourceState(ctx, workflow.ID, "scan-source", "fp-typed-slice")
|
|
if err != nil {
|
|
t.Fatalf("get source state: %v", err)
|
|
}
|
|
if state == nil {
|
|
t.Fatalf("expected committed source state")
|
|
}
|
|
if len(state.SeenFingerprints) != 3 {
|
|
t.Fatalf("expected 3 seen fingerprints, got %#v", state.SeenFingerprints)
|
|
}
|
|
if strings.Join(state.SeenFingerprints, ",") != "a,b,c" {
|
|
t.Fatalf("unexpected seen fingerprints %#v", state.SeenFingerprints)
|
|
}
|
|
}
|
|
|
|
func writeFile(t *testing.T, path string, content string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatalf("write %s: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func createRosbagDB3(t *testing.T, path string) {
|
|
t.Helper()
|
|
command := exec.Command(
|
|
"python3",
|
|
"-c",
|
|
strings.Join([]string{
|
|
"import sqlite3, sys",
|
|
"conn = sqlite3.connect(sys.argv[1])",
|
|
"cur = conn.cursor()",
|
|
"cur.execute('create table topics (id integer primary key, name text, type text, serialization_format text)')",
|
|
"cur.execute('create table messages (id integer primary key, topic_id integer, timestamp integer, data blob)')",
|
|
"cur.execute(\"insert into topics (id, name, type, serialization_format) values (1, '/camera', 'sensor_msgs/msg/Image', 'cdr')\")",
|
|
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 100, X'01')\")",
|
|
"cur.execute(\"insert into messages (topic_id, timestamp, data) values (1, 200, X'02')\")",
|
|
"conn.commit()",
|
|
"conn.close()",
|
|
}, "; "),
|
|
path,
|
|
)
|
|
if output, err := command.CombinedOutput(); err != nil {
|
|
t.Fatalf("create rosbag db3 %s: %v: %s", path, err, string(output))
|
|
}
|
|
}
|
|
|
|
func TestChooseImagePrefersRuntimeOperatorRunnerForBuiltins(t *testing.T) {
|
|
node := domain.WorkflowNode{
|
|
Data: domain.WorkflowNodeData{
|
|
Image: "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:old-tag",
|
|
},
|
|
}
|
|
image := chooseImage(node, "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag")
|
|
if image != "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/droboticflow-operator-runner:new-tag" {
|
|
t.Fatalf("expected runtime image override for built-in operator-runner, got %q", image)
|
|
}
|
|
}
|
|
|
|
func TestChooseImagePrefersDigestPinnedOperatorImageID(t *testing.T) {
|
|
node := domain.WorkflowNode{
|
|
Data: domain.WorkflowNodeData{
|
|
Image: "ccr.example/dataflow-poc/droboticflow-operator-openvla-finetune:old",
|
|
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-openvla-finetune:fixed@sha256:" + strings.Repeat("a", 64),
|
|
},
|
|
}
|
|
image := chooseImage(node, "ccr.example/dataflow-poc/droboticflow-operator-runner:live")
|
|
if image != node.Data.OperatorImageID {
|
|
t.Fatalf("expected digest-pinned operator image id, got %q", image)
|
|
}
|
|
}
|
|
|
|
func TestChooseImagePrefersUntaggedDigestPinnedOperatorImageID(t *testing.T) {
|
|
node := domain.WorkflowNode{
|
|
Data: domain.WorkflowNodeData{
|
|
Image: "ccr.example/dataflow-poc/droboticflow-operator-act-policy-train:built-tag",
|
|
OperatorImageID: "ccr.example/dataflow-poc/droboticflow-operator-act-policy-train@sha256:" + strings.Repeat("b", 64),
|
|
},
|
|
}
|
|
image := chooseImage(node, "ccr.example/dataflow-poc/droboticflow-operator-runner:live")
|
|
if image != node.Data.OperatorImageID {
|
|
t.Fatalf("expected untagged digest-pinned operator image id, got %q", image)
|
|
}
|
|
}
|
|
|
|
func createTestVideoMP4(t *testing.T, path string, pattern string, fps int, frames int, width int, height int) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
|
|
}
|
|
command := exec.Command(
|
|
"python3",
|
|
"-c",
|
|
strings.Join([]string{
|
|
"import cv2, numpy as np, sys",
|
|
"path, pattern, fps, frames, width, height = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]), int(sys.argv[6])",
|
|
"fourcc = cv2.VideoWriter_fourcc(*'mp4v')",
|
|
"writer = cv2.VideoWriter(path, fourcc, fps, (width, height))",
|
|
"assert writer.isOpened(), 'video writer did not open'",
|
|
"for index in range(frames):",
|
|
" frame = np.zeros((height, width, 3), dtype=np.uint8)",
|
|
" if pattern == 'sharp':",
|
|
" x = int((index * 3) % max(1, width - 18))",
|
|
" cv2.rectangle(frame, (x, 8), (x + 16, 24), (255, 255, 255), -1)",
|
|
" cv2.putText(frame, str(index % 10), (6, height - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1, cv2.LINE_AA)",
|
|
" elif pattern == 'blur':",
|
|
" cv2.rectangle(frame, (16, 16), (width - 16, height - 16), (220, 220, 220), -1)",
|
|
" frame = cv2.GaussianBlur(frame, (23, 23), 0)",
|
|
" else:",
|
|
" x = int((index * 5) % max(1, width - 20))",
|
|
" frame[:] = (40, 60, 90)",
|
|
" cv2.circle(frame, (x + 10, height // 2), 10, (80, 140, 190), -1)",
|
|
" writer.write(frame)",
|
|
"writer.release()",
|
|
"capture = cv2.VideoCapture(path)",
|
|
"assert capture.isOpened(), 'video capture did not open'",
|
|
"capture.release()",
|
|
"print(path)",
|
|
}, "\n"),
|
|
path,
|
|
pattern,
|
|
fmt.Sprintf("%d", fps),
|
|
fmt.Sprintf("%d", frames),
|
|
fmt.Sprintf("%d", width),
|
|
fmt.Sprintf("%d", height),
|
|
)
|
|
if output, err := command.CombinedOutput(); err != nil {
|
|
t.Fatalf("create test mp4 %s: %v: %s", path, err, string(output))
|
|
}
|
|
}
|
|
|
|
func createXPlatformDataset(t *testing.T) string {
|
|
t.Helper()
|
|
root := filepath.Join(t.TempDir(), "algo", "X-platform-test")
|
|
fps := 10
|
|
width := 96
|
|
height := 64
|
|
|
|
type videoSpec struct {
|
|
Name string
|
|
Pattern string
|
|
Frames int
|
|
Duration float64
|
|
}
|
|
type collectionSpec struct {
|
|
Name string
|
|
Type string
|
|
Place string
|
|
Date string
|
|
DeviceID string
|
|
HasIntrinsics bool
|
|
VideoDirectory string
|
|
Videos []videoSpec
|
|
}
|
|
|
|
collections := []collectionSpec{
|
|
{
|
|
Name: "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
Type: "家庭",
|
|
Place: "蓟门里小区",
|
|
Date: "20260319",
|
|
DeviceID: "C00000",
|
|
HasIntrinsics: true,
|
|
VideoDirectory: "DJI_001",
|
|
Videos: []videoSpec{
|
|
{Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 40, Duration: 4.0},
|
|
{Name: "DJI_0002_D.mp4", Pattern: "blur", Frames: 20, Duration: 2.0},
|
|
},
|
|
},
|
|
{
|
|
Name: "BJ_002_0002_OsmoNano_2026-03-20_10-30-00",
|
|
Type: "办公",
|
|
Place: "中关村",
|
|
Date: "20260320",
|
|
DeviceID: "C00001",
|
|
HasIntrinsics: false,
|
|
VideoDirectory: "DJI_001",
|
|
Videos: []videoSpec{
|
|
{Name: "DJI_0001_D.mp4", Pattern: "sharp", Frames: 20, Duration: 2.0},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, collection := range collections {
|
|
collectionRoot := filepath.Join(root, collection.Name)
|
|
videoRoot := filepath.Join(collectionRoot, collection.VideoDirectory)
|
|
videoMetaRows := make([]map[string]any, 0, len(collection.Videos))
|
|
for index, video := range collection.Videos {
|
|
videoPath := filepath.Join(videoRoot, video.Name)
|
|
createTestVideoMP4(t, videoPath, video.Pattern, fps, video.Frames, width, height)
|
|
videoMetaRows = append(videoMetaRows, map[string]any{
|
|
"filename": video.Name,
|
|
"codec": "mp4v",
|
|
"resolution": fmt.Sprintf("%dx%d", width, height),
|
|
"frameRate": float64(fps),
|
|
"bitrate": 2000000 + index*100000,
|
|
"duration": video.Duration,
|
|
"startTime": 1773904218 + index*60,
|
|
})
|
|
}
|
|
metaJSON, err := json.Marshal(map[string]any{
|
|
"type": collection.Type,
|
|
"place": collection.Place,
|
|
"mission": "无",
|
|
"project_id": collection.Name,
|
|
"date": collection.Date,
|
|
"device_id": collection.DeviceID,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal meta json: %v", err)
|
|
}
|
|
writeFile(t, filepath.Join(collectionRoot, "meta.json"), string(metaJSON))
|
|
videoMetaJSON, err := json.Marshal(videoMetaRows)
|
|
if err != nil {
|
|
t.Fatalf("marshal video_meta json: %v", err)
|
|
}
|
|
writeFile(t, filepath.Join(collectionRoot, "video_meta.json"), string(videoMetaJSON))
|
|
writeFile(t, filepath.Join(videoRoot, "DJI_meta.json"), `{"ok":true}`)
|
|
if collection.HasIntrinsics {
|
|
writeFile(t, filepath.Join(collectionRoot, "intrinsics.json"), `{"fx": 1000, "fy": 1000}`)
|
|
}
|
|
}
|
|
|
|
return root
|
|
}
|
|
|
|
func waitForRunStatus(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected domain.RunStatus, timeout time.Duration) *domain.RunRecord {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
if run != nil && run.Status == expected {
|
|
return run
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
t.Fatalf("timed out waiting for run %s to become %s, got %#v", runID, expected, run)
|
|
return nil
|
|
}
|
|
|
|
func waitForRunningNodeCount(t *testing.T, ctx context.Context, dataStore *store.MemoryStore, runID string, expected int, timeout time.Duration) *domain.RunRecord {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
if run == nil {
|
|
time.Sleep(100 * time.Millisecond)
|
|
continue
|
|
}
|
|
running := 0
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.Status == domain.NodeExecutionStatusRunning {
|
|
running++
|
|
}
|
|
}
|
|
if running == expected {
|
|
return run
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
run, err := dataStore.GetRun(ctx, runID)
|
|
if err != nil {
|
|
t.Fatalf("get run %s: %v", runID, err)
|
|
}
|
|
t.Fatalf("timed out waiting for run %s to have %d running nodes, got %#v", runID, expected, run)
|
|
return nil
|
|
}
|
|
|
|
func TestRuntimeProcessesBatchRunToCompletion(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Worker 项目", Description: "worker"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "批处理工作流",
|
|
Description: "complete path",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "OSS 数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Inputs: []domain.PortDefinition{},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-1", Name: "file", Type: domain.PortTypeFile},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "INLINE"},
|
|
{Key: "SOURCE_TEXT", Value: "worker-runtime-test"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "sink",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "sink",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{},
|
|
},
|
|
Position: domain.Position{X: 100, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "source", SourceHandle: "out-1", Target: "sink", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed run, got %#v", completed)
|
|
}
|
|
for _, item := range completed.NodeExecutions {
|
|
if item.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected completed node execution, got %s", item.Status)
|
|
}
|
|
}
|
|
if len(completed.Artifacts) == 0 {
|
|
t.Fatalf("expected artifacts")
|
|
}
|
|
if completed.Artifacts[0].Path == "" {
|
|
t.Fatalf("expected real artifact path")
|
|
}
|
|
if len(completed.ResourceSeries) == 0 {
|
|
t.Fatalf("expected resource samples from operator runner")
|
|
}
|
|
}
|
|
|
|
func TestRuntimeStopsGracefullyWithoutDispatchingNewNodes(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "停止项目", Description: "graceful"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "停止工作流",
|
|
Description: "graceful stop path",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxConcurrentRuns: 1,
|
|
NodeTimeoutSeconds: 30,
|
|
RunTimeoutSeconds: 60,
|
|
MaxRetryAttempts: 0,
|
|
RetryBackoffSeconds: 0,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "payload", Name: "payload", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "COMMAND", Value: `sleep 1; printf '{"value":1}' > payload.json`},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "sink",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "sink",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "input", Name: "input", Type: domain.PortTypeJSON},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 120, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "source", SourceHandle: "payload", Target: "sink", TargetHandle: "input"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, processErr := runtime.ProcessOnce(ctx)
|
|
done <- processErr
|
|
}()
|
|
|
|
waitForRunningNodeCount(t, ctx, dataStore, run.ID, 1, 5*time.Second)
|
|
stored, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get running run: %v", err)
|
|
}
|
|
stored.Status = domain.RunStatus("stopping")
|
|
stored.StopMode = domain.RunStopModeGraceful
|
|
stored.StopRequestedAt = nowISO()
|
|
stored.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *stored); err != nil {
|
|
t.Fatalf("request graceful stop: %v", err)
|
|
}
|
|
|
|
select {
|
|
case err := <-done:
|
|
if err != nil {
|
|
t.Fatalf("process run: %v", err)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("timed out waiting for graceful stop")
|
|
}
|
|
|
|
finalRun, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get final run: %v", err)
|
|
}
|
|
if finalRun == nil || finalRun.Status != domain.RunStatusCancelled {
|
|
t.Fatalf("expected cancelled run after graceful stop, got %#v", finalRun)
|
|
}
|
|
if sink := mustNodeExecution(t, finalRun, "source"); sink.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected source node to complete before stop, got %s", sink.Status)
|
|
}
|
|
for _, item := range finalRun.NodeExecutions {
|
|
if item.NodeID == "sink" {
|
|
t.Fatalf("expected sink node not to start after graceful stop, got %#v", item)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimePassesContainerCommandAndArgsToLauncher(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "镜像项目", Description: "command args"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "自定义容器入口",
|
|
Description: "verify command args",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "custom",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "ACT 策略训练",
|
|
Description: "custom entrypoint",
|
|
Image: "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime",
|
|
ContainerCommand: []string{"/bin/bash", "-lc"},
|
|
ContainerArgs: []string{"python /workspace/operators/act.py"},
|
|
RuntimeContract: map[string]any{"handler": "builtin_act_policy_train"},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
var captured k8s.LaunchInput
|
|
runtime := &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "operator-runner:latest",
|
|
workspaceRoot: t.TempDir(),
|
|
launcher: testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
captured = input
|
|
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
|
|
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: "custom", Namespace: "droboticflow"},
|
|
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
|
}, nil
|
|
},
|
|
},
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
if captured.Image != "pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime" {
|
|
t.Fatalf("image = %q", captured.Image)
|
|
}
|
|
if strings.Join(captured.Command, " ") != "/bin/bash -lc" {
|
|
t.Fatalf("command = %#v", captured.Command)
|
|
}
|
|
if len(captured.Args) != 1 || captured.Args[0] != "python /workspace/operators/act.py" {
|
|
t.Fatalf("args = %#v", captured.Args)
|
|
}
|
|
if captured.Env["DROBOTICFLOW_OPERATOR_HANDLER"] != "act_policy_train" {
|
|
t.Fatalf("handler env = %q", captured.Env["DROBOTICFLOW_OPERATOR_HANDLER"])
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed run, got %#v", completed)
|
|
}
|
|
}
|
|
|
|
func TestRuntimePassesLightweightOperatorMountToLauncher(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "轻量项目", Description: "lightweight"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "轻量算子工作流",
|
|
Description: "verify lightweight code mount",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "light",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "轻量算子",
|
|
Description: "code from workspace",
|
|
Image: "python:3.11-slim",
|
|
ContainerCommand: []string{"/bin/sh", "-lc"},
|
|
ContainerArgs: []string{"python operator.py"},
|
|
RuntimeContract: map[string]any{
|
|
"lightweightOperator": map[string]any{
|
|
"enabled": true,
|
|
"codeSubPath": "lightweight-operators/demo",
|
|
"mountPath": "/app/operator",
|
|
"mounts": []any{map[string]any{"name": "data", "pvcName": "dataset-pvc", "subPath": "robotwin", "mountPath": "/data", "readOnly": true}},
|
|
},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
var captured k8s.LaunchInput
|
|
runtime := &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "operator-runner:latest",
|
|
workspaceRoot: t.TempDir(),
|
|
launcher: testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
captured = input
|
|
if path := strings.TrimSpace(input.Env["STAGE_MANIFEST_PATH"]); path != "" {
|
|
if err := os.WriteFile(path, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: "light", Namespace: "droboticflow"},
|
|
Output: `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
|
}, nil
|
|
},
|
|
},
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
if captured.CodeSubPath != "lightweight-operators/demo" || captured.CodeMountPath != "/app/operator" {
|
|
t.Fatalf("unexpected lightweight mount: subPath=%q mountPath=%q", captured.CodeSubPath, captured.CodeMountPath)
|
|
}
|
|
if captured.Env["LIGHTWEIGHT_OPERATOR_CODE_DIR"] != "/app/operator" {
|
|
t.Fatalf("LIGHTWEIGHT_OPERATOR_CODE_DIR = %q", captured.Env["LIGHTWEIGHT_OPERATOR_CODE_DIR"])
|
|
}
|
|
if len(captured.PVCMounts) != 1 || captured.PVCMounts[0].PVCName != "dataset-pvc" || captured.PVCMounts[0].MountPath != "/data" || !captured.PVCMounts[0].ReadOnly {
|
|
t.Fatalf("unexpected additional PVC mounts: %#v", captured.PVCMounts)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeRejectsLightweightOperatorWithoutExplicitContainerArgs(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "轻量项目", Description: "lightweight"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "轻量算子工作流",
|
|
Description: "missing explicit args",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "light",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "轻量算子",
|
|
Description: "code from workspace",
|
|
Image: "python:3.11-slim",
|
|
ContainerCommand: []string{"python", "/app/operator/operator.py"},
|
|
RuntimeContract: map[string]any{
|
|
"lightweightOperator": map[string]any{
|
|
"enabled": true,
|
|
"codeSubPath": "lightweight-operators/demo",
|
|
"mountPath": "/app/operator",
|
|
},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
launchCalled := false
|
|
runtime := &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "operator-runner:latest",
|
|
workspaceRoot: t.TempDir(),
|
|
launcher: testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
launchCalled = true
|
|
return nil, fmt.Errorf("launch should not be called")
|
|
},
|
|
},
|
|
}
|
|
processed := mustProcessOnceRun(t, runtime, ctx)
|
|
if launchCalled {
|
|
t.Fatalf("expected lightweight preflight to fail before launch")
|
|
}
|
|
if processed.Status != domain.RunStatusFailed || !strings.Contains(processed.Error, "containerArgs must be explicitly configured") {
|
|
t.Fatalf("expected explicit args preflight failure, got status=%s error=%q", processed.Status, processed.Error)
|
|
}
|
|
}
|
|
|
|
func TestRuntimePausesForAuditAndResumes(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Audit 项目", Description: "audit"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "审核工作流",
|
|
Description: "audit path",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "review",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "人工审核 (Human Review)",
|
|
Category: "HUMAN",
|
|
Description: "review",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "approved", Name: "审核通过", Type: domain.PortTypeJSON},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "sink",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "json", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{},
|
|
},
|
|
Position: domain.Position{X: 100, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "review", SourceHandle: "approved", Target: "upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
paused, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get paused run: %v", err)
|
|
}
|
|
if paused == nil || paused.Status != domain.RunStatusWaitingForAudit {
|
|
t.Fatalf("expected waiting_for_audit run, got %#v", paused)
|
|
}
|
|
|
|
tasks, err := dataStore.ListAuditTasks(ctx, "all")
|
|
if err != nil {
|
|
t.Fatalf("list audit tasks: %v", err)
|
|
}
|
|
var task *domain.AuditTaskRecord
|
|
for _, item := range tasks {
|
|
if item.RunID == run.ID && item.Status == domain.AuditStatusPending {
|
|
copied := item
|
|
task = &copied
|
|
break
|
|
}
|
|
}
|
|
if task == nil {
|
|
t.Fatalf("expected pending audit task")
|
|
}
|
|
if task.OrganizationID != run.OrganizationID {
|
|
t.Fatalf("audit task organization = %q, want run organization %q", task.OrganizationID, run.OrganizationID)
|
|
}
|
|
|
|
task.Status = domain.AuditStatusApproved
|
|
task.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertAuditTask(ctx, *task); err != nil {
|
|
t.Fatalf("approve audit task: %v", err)
|
|
}
|
|
paused.Status = domain.RunStatusQueued
|
|
paused.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *paused); err != nil {
|
|
t.Fatalf("queue paused run: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("resume process once: %v", err)
|
|
}
|
|
|
|
resumed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get resumed run: %v", err)
|
|
}
|
|
if resumed == nil || resumed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed resumed run, got %#v", resumed)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeRoutesArtifactsByOutputHandle(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Route 项目", Description: "route"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "条件分流工作流",
|
|
Description: "route by handle",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-manifest", Name: "manifest", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "COMMAND", Value: `printf '{"items":[{"kind":"keep","value":1},{"kind":"drop","value":2}]}' > routed.json`},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "route",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "条件分流",
|
|
Category: "LOGIC",
|
|
Description: "route",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in-1", Name: "input", Type: domain.PortTypeJSON},
|
|
},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "matched", Name: "matched", Type: domain.PortTypeJSON},
|
|
{ID: "unmatched", Name: "unmatched", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "FIELD_PATH", Value: "kind"},
|
|
{Key: "OPERATOR", Value: "eq"},
|
|
{Key: "EXPECTED_JSON", Value: `"keep"`},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 200, Y: 0},
|
|
},
|
|
{
|
|
ID: "keep-upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "keep sink",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 420, Y: -80},
|
|
},
|
|
{
|
|
ID: "drop-upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
Category: "IO",
|
|
Description: "drop sink",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 420, Y: 80},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-route", Source: "source", SourceHandle: "out-manifest", Target: "route", TargetHandle: "in-1"},
|
|
{ID: "route-keep", Source: "route", SourceHandle: "matched", Target: "keep-upload", TargetHandle: "in-1"},
|
|
{ID: "route-drop", Source: "route", SourceHandle: "unmatched", Target: "drop-upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed run, got %#v", completed)
|
|
}
|
|
|
|
routeExecution := mustNodeExecution(t, completed, "route")
|
|
matched := mustArtifact(t, routeExecution, "matched.json")
|
|
unmatched := mustArtifact(t, routeExecution, "unmatched.json")
|
|
if matched.PortID != "matched" || unmatched.PortID != "unmatched" {
|
|
t.Fatalf("expected output port IDs on split artifacts, got matched=%q unmatched=%q", matched.PortID, unmatched.PortID)
|
|
}
|
|
|
|
keepUpload := mustNodeExecution(t, completed, "keep-upload")
|
|
dropUpload := mustNodeExecution(t, completed, "drop-upload")
|
|
keepSummary := mustArtifact(t, keepUpload, "uploaded-results.json")
|
|
dropSummary := mustArtifact(t, dropUpload, "uploaded-results.json")
|
|
keepSummaryPayload := mustArtifactPayload(t, keepSummary)
|
|
dropSummaryPayload := mustArtifactPayload(t, dropSummary)
|
|
keepItems, _ := keepSummaryPayload["items"].([]any)
|
|
dropItems, _ := dropSummaryPayload["items"].([]any)
|
|
if len(keepItems) != 1 || len(dropItems) != 1 {
|
|
t.Fatalf("expected one routed payload per sink, got keep=%d drop=%d", len(keepItems), len(dropItems))
|
|
}
|
|
|
|
keepPayload, _ := keepItems[0].(map[string]any)
|
|
dropPayload, _ := dropItems[0].(map[string]any)
|
|
keepRows, _ := keepPayload["items"].([]any)
|
|
dropRows, _ := dropPayload["items"].([]any)
|
|
if len(keepRows) != 1 || len(dropRows) != 1 {
|
|
t.Fatalf("expected one row in each routed payload, got keep=%d drop=%d", len(keepRows), len(dropRows))
|
|
}
|
|
if keepRow, _ := keepRows[0].(map[string]any); keepRow != nil {
|
|
if keepRow["kind"] != "keep" {
|
|
t.Fatalf("unexpected keep row: %#v", keepRow)
|
|
}
|
|
}
|
|
if dropRow, _ := dropRows[0].(map[string]any); dropRow != nil {
|
|
if dropRow["kind"] != "drop" {
|
|
t.Fatalf("unexpected drop row: %#v", dropRow)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeSupportsPreviewAndDatasetBuiltins(t *testing.T) {
|
|
t.Run("command previews text and csv outputs", func(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Preview 项目", Description: "preview"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "文件预览工作流",
|
|
Description: "preview files",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "COMMAND", Value: "printf 'name,score\\nalpha,1\\nbeta,2\\n' > report.csv && printf 'hello\\nworld\\n' > notes.txt"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
commandExecution := mustNodeExecution(t, completed, "command")
|
|
report := mustArtifact(t, commandExecution, "report.csv")
|
|
notes := mustArtifact(t, commandExecution, "notes.txt")
|
|
reportPreview, _ := report.Data["preview"].(map[string]any)
|
|
notesPreview, _ := notes.Data["preview"].(map[string]any)
|
|
if reportPreview["kind"] != "table" {
|
|
t.Fatalf("expected table preview for csv artifact, got %#v", reportPreview)
|
|
}
|
|
if notesPreview["kind"] != "text" {
|
|
t.Fatalf("expected text preview for text artifact, got %#v", notesPreview)
|
|
}
|
|
})
|
|
|
|
t.Run("lerobot dataset summary", func(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
datasetRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(datasetRoot, "meta", "info.json"), `{"robot_type":"aloha","fps":30}`)
|
|
writeFile(t, filepath.Join(datasetRoot, "meta", "stats.json"), `{"episodes":2}`)
|
|
writeFile(t, filepath.Join(datasetRoot, "meta", "tasks.jsonl"), "{\"task\":\"pick\"}\n{\"task\":\"place\"}\n")
|
|
writeFile(t, filepath.Join(datasetRoot, "data", "episode_000.parquet"), "PAR1")
|
|
writeFile(t, filepath.Join(datasetRoot, "videos", "episode_000.mp4"), "video")
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "LeRobot 项目", Description: "lerobot"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "LeRobot 概览",
|
|
Description: "lerobot summary",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: datasetRoot},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "summary",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "LeRobot 数据集概览",
|
|
Category: "ROBOTICS",
|
|
Description: "lerobot",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 240, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "lerobot-summary.json")
|
|
summaryPayload := mustArtifactPayload(t, summary)
|
|
preview, _ := summaryPayload["preview"].(map[string]any)
|
|
if preview["format"] != "lerobot" {
|
|
t.Fatalf("expected lerobot preview, got %#v", preview)
|
|
}
|
|
if summaryPayload["datasetType"] != "lerobot" {
|
|
t.Fatalf("expected datasetType lerobot, got %#v", summaryPayload)
|
|
}
|
|
})
|
|
|
|
t.Run("rosbag sqlite summary", func(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
datasetRoot := t.TempDir()
|
|
dbPath := filepath.Join(datasetRoot, "bag_0.db3")
|
|
createRosbagDB3(t, dbPath)
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "ROS 项目", Description: "rosbag"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "ROSBag 概览",
|
|
Description: "rosbag summary",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: dbPath},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "summary",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "ROSBag 数据概览",
|
|
Category: "ROBOTICS",
|
|
Description: "rosbag",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "summary", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "SOURCE_PATH", Value: dbPath}},
|
|
},
|
|
Position: domain.Position{X: 240, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-summary", Source: "source", SourceHandle: "out-1", Target: "summary", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
summary := mustArtifact(t, mustNodeExecution(t, completed, "summary"), "rosbag-summary.json")
|
|
summaryPayload := mustArtifactPayload(t, summary)
|
|
preview, _ := summaryPayload["preview"].(map[string]any)
|
|
if preview["format"] != "rosbag2" {
|
|
t.Fatalf("expected rosbag2 preview, got %#v", preview)
|
|
}
|
|
topics, _ := preview["topics"].([]any)
|
|
if len(topics) != 1 {
|
|
t.Fatalf("expected one topic summary, got %#v", topics)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRuntimeProcessesXPlatformPipeline(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
datasetRoot := createXPlatformDataset(t)
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "XPlatform 项目", Description: "x-platform"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "XPlatform 处理流水线",
|
|
Description: "scan -> validate -> quality -> filter -> slice -> annotate -> package -> sample",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: datasetRoot},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "index",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "X 平台数据集索引",
|
|
Category: "DATASET",
|
|
Description: "index dataset",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "dataset", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 220, Y: 0},
|
|
},
|
|
{
|
|
ID: "validate",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "采集目录校验",
|
|
Category: "DATASET",
|
|
Description: "validate dataset",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "validated", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 440, Y: 0},
|
|
},
|
|
{
|
|
ID: "quality",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "视频质检",
|
|
Category: "VIDEO",
|
|
Description: "quality",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "quality", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "MIN_CLARITY", Value: "0.8"},
|
|
{Key: "MAX_SHAKE", Value: "0.1"},
|
|
{Key: "MAX_HAND_OCCLUSION", Value: "0.1"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 660, Y: 0},
|
|
},
|
|
{
|
|
ID: "filter",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "数据过滤",
|
|
Category: "VIDEO",
|
|
Description: "filter dataset",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "filtered", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATE_START", Value: "2026-02-01"},
|
|
{Key: "DATE_END", Value: "2026-04-05"},
|
|
{Key: "PLACE", Value: "蓟门里小区"},
|
|
{Key: "TYPE", Value: "家庭"},
|
|
{Key: "MIN_CLARITY", Value: "0.8"},
|
|
{Key: "MAX_SHAKE", Value: "0.1"},
|
|
{Key: "MAX_HAND_OCCLUSION", Value: "0.1"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 880, Y: 0},
|
|
},
|
|
{
|
|
ID: "slice",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "视频切片",
|
|
Category: "VIDEO",
|
|
Description: "slice video",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "clips", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "SEGMENT_DURATION_SECONDS", Value: "2"}},
|
|
},
|
|
Position: domain.Position{X: 1100, Y: 0},
|
|
},
|
|
{
|
|
ID: "annotate",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "视频标注",
|
|
Category: "VIDEO",
|
|
Description: "annotate clips",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "FRAME_INTERVAL_SECONDS", Value: "0.5"}},
|
|
},
|
|
Position: domain.Position{X: 1320, Y: 0},
|
|
},
|
|
{
|
|
ID: "package",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "成品打包",
|
|
Category: "VIDEO",
|
|
Description: "package results",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "packages", Type: domain.PortTypeJSON}},
|
|
},
|
|
Position: domain.Position{X: 1540, Y: 0},
|
|
},
|
|
{
|
|
ID: "sample",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "抽检采样",
|
|
Category: "QA",
|
|
Description: "sample packages",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "RULES_JSON", Value: `[{"id":"clarity-check","field":"quality.clarity","operator":"lt","value":1,"samplePercent":100},{"id":"confidence-check","field":"annotation.minConfidence","operator":"lt","value":1,"samplePercent":100}]`},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 1760, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-index", Source: "source", SourceHandle: "out-1", Target: "index", TargetHandle: "in-1"},
|
|
{ID: "index-validate", Source: "index", SourceHandle: "out-1", Target: "validate", TargetHandle: "in-1"},
|
|
{ID: "validate-quality", Source: "validate", SourceHandle: "out-1", Target: "quality", TargetHandle: "in-1"},
|
|
{ID: "quality-filter", Source: "quality", SourceHandle: "out-1", Target: "filter", TargetHandle: "in-1"},
|
|
{ID: "filter-slice", Source: "filter", SourceHandle: "out-1", Target: "slice", TargetHandle: "in-1"},
|
|
{ID: "slice-annotate", Source: "slice", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"},
|
|
{ID: "annotate-package", Source: "annotate", SourceHandle: "out-1", Target: "package", TargetHandle: "in-1"},
|
|
{ID: "package-sample", Source: "package", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected completed run, got %#v", completed)
|
|
}
|
|
|
|
indexArtifact := mustArtifact(t, mustNodeExecution(t, completed, "index"), "dataset-index.json")
|
|
indexPayload := mustArtifactPayload(t, indexArtifact)
|
|
if indexPayload["collectionCount"] != float64(2) {
|
|
t.Fatalf("expected collectionCount=2, got %#v", indexPayload)
|
|
}
|
|
if indexPayload["videoCount"] != float64(3) {
|
|
t.Fatalf("expected videoCount=3, got %#v", indexPayload)
|
|
}
|
|
|
|
validateArtifact := mustArtifact(t, mustNodeExecution(t, completed, "validate"), "dataset-validation.json")
|
|
validatePayload := mustArtifactPayload(t, validateArtifact)
|
|
if validatePayload["validCollectionCount"] != float64(2) {
|
|
t.Fatalf("expected validCollectionCount=2, got %#v", validatePayload)
|
|
}
|
|
|
|
qualityArtifact := mustArtifact(t, mustNodeExecution(t, completed, "quality"), "quality-report.json")
|
|
qualityPayload := mustArtifactPayload(t, qualityArtifact)
|
|
if qualityPayload["videoCount"] != float64(3) {
|
|
t.Fatalf("expected quality videoCount=3, got %#v", qualityPayload)
|
|
}
|
|
if qualityPayload["passedCount"] != float64(2) {
|
|
t.Fatalf("expected passedCount=2, got %#v", qualityPayload)
|
|
}
|
|
|
|
filterArtifact := mustArtifact(t, mustNodeExecution(t, completed, "filter"), "filtered-videos.json")
|
|
filterPayload := mustArtifactPayload(t, filterArtifact)
|
|
if filterPayload["count"] != float64(1) {
|
|
t.Fatalf("expected filter count=1, got %#v", filterPayload)
|
|
}
|
|
|
|
sliceExecution := mustNodeExecution(t, completed, "slice")
|
|
clipsArtifact := mustArtifact(t, sliceExecution, "clips-manifest.json")
|
|
clipsPayload := mustArtifactPayload(t, clipsArtifact)
|
|
if clipsPayload["clipCount"] != float64(2) {
|
|
t.Fatalf("expected clipCount=2, got %#v", clipsPayload)
|
|
}
|
|
clipVideos := 0
|
|
for _, artifact := range sliceExecution.Artifacts {
|
|
if strings.HasSuffix(artifact.Name, ".mp4") {
|
|
clipVideos++
|
|
}
|
|
}
|
|
if clipVideos != 2 {
|
|
t.Fatalf("expected 2 clip videos, got %d", clipVideos)
|
|
}
|
|
|
|
annotateExecution := mustNodeExecution(t, completed, "annotate")
|
|
annotationsArtifact := mustArtifact(t, annotateExecution, "annotations-manifest.json")
|
|
annotationsPayload := mustArtifactPayload(t, annotationsArtifact)
|
|
if annotationsPayload["annotationCount"] != float64(2) {
|
|
t.Fatalf("expected annotationCount=2, got %#v", annotationsPayload)
|
|
}
|
|
|
|
packageExecution := mustNodeExecution(t, completed, "package")
|
|
packagesArtifact := mustArtifact(t, packageExecution, "packages-manifest.json")
|
|
packagesPayload := mustArtifactPayload(t, packagesArtifact)
|
|
if packagesPayload["packageCount"] != float64(2) {
|
|
t.Fatalf("expected packageCount=2, got %#v", packagesPayload)
|
|
}
|
|
packageFiles := 0
|
|
for _, artifact := range packageExecution.Artifacts {
|
|
if strings.Contains(artifact.Path, "/packages/") && (strings.HasSuffix(artifact.Name, ".json") || strings.HasSuffix(artifact.Name, ".mp4")) {
|
|
packageFiles++
|
|
}
|
|
}
|
|
if packageFiles < 8 {
|
|
t.Fatalf("expected packaged files to be emitted, got %d artifacts", packageFiles)
|
|
}
|
|
|
|
sampleArtifact := mustArtifact(t, mustNodeExecution(t, completed, "sample"), "audit-samples.json")
|
|
samplePayload := mustArtifactPayload(t, sampleArtifact)
|
|
if samplePayload["sampledCount"] != float64(2) {
|
|
t.Fatalf("expected sampledCount=2, got %#v", samplePayload)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeCommitsIncrementalSourceStateOnlyOnSuccessfulRuns(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
t.Run("commits state after successful run and next run only emits new files", func(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "增量项目", Description: "incremental"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "增量源工作流",
|
|
Description: "incremental source",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create first run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process first run: %v", err)
|
|
}
|
|
|
|
completedOne, err := dataStore.GetRun(ctx, runOne.ID)
|
|
if err != nil {
|
|
t.Fatalf("get first run: %v", err)
|
|
}
|
|
manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json")
|
|
manifestOnePayload := mustArtifactPayload(t, manifestOne)
|
|
if manifestOnePayload["newCount"] != float64(1) {
|
|
t.Fatalf("expected newCount=1 on first run, got %#v", manifestOnePayload)
|
|
}
|
|
|
|
state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestOnePayload["sourceFingerprint"].(string)))
|
|
if err != nil {
|
|
t.Fatalf("get source state: %v", err)
|
|
}
|
|
if state == nil || len(state.SeenFingerprints) != 1 {
|
|
t.Fatalf("expected committed source state after first run, got %#v", state)
|
|
}
|
|
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`)
|
|
runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
|
|
if err != nil {
|
|
t.Fatalf("create second run: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process second run: %v", err)
|
|
}
|
|
|
|
completedTwo, err := dataStore.GetRun(ctx, runTwo.ID)
|
|
if err != nil {
|
|
t.Fatalf("get second run: %v", err)
|
|
}
|
|
manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json")
|
|
manifestTwoPayload := mustArtifactPayload(t, manifestTwo)
|
|
if manifestTwoPayload["listedCount"] != float64(2) {
|
|
t.Fatalf("expected listedCount=2 on second run, got %#v", manifestTwoPayload)
|
|
}
|
|
if manifestTwoPayload["newCount"] != float64(1) {
|
|
t.Fatalf("expected newCount=1 on second run, got %#v", manifestTwoPayload)
|
|
}
|
|
entries, ok := manifestTwoPayload["entries"].([]any)
|
|
if !ok || len(entries) != 1 {
|
|
t.Fatalf("expected one emitted entry on second run, got %#v", manifestTwoPayload["entries"])
|
|
}
|
|
entry, ok := entries[0].(map[string]any)
|
|
if !ok || !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/meta.json") {
|
|
t.Fatalf("expected second run to emit only newly added file, got %#v", entries[0])
|
|
}
|
|
|
|
state, err = dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string)))
|
|
if err != nil {
|
|
t.Fatalf("get source state after second run: %v", err)
|
|
}
|
|
if state == nil || len(state.SeenFingerprints) != 2 {
|
|
t.Fatalf("expected 2 seen fingerprints after second run, got %#v", state)
|
|
}
|
|
if state.LastNewCount != 1 || state.LastListedCount != 2 {
|
|
t.Fatalf("unexpected source state stats after second run: %#v", state)
|
|
}
|
|
})
|
|
|
|
t.Run("directory incremental mode emits entire newly discovered directories", func(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "目录增量项目", Description: "incremental-directory"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-a.mp4"), "a")
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "目录增量源工作流",
|
|
Description: "incremental directory source",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
|
|
{Key: "INCREMENTAL_UNIT", Value: "DIRECTORY"},
|
|
{Key: "INCREMENTAL_DIRECTORY_DEPTH", Value: "1"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
runOne, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create first run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process first run: %v", err)
|
|
}
|
|
|
|
completedOne, err := dataStore.GetRun(ctx, runOne.ID)
|
|
if err != nil {
|
|
t.Fatalf("get first run: %v", err)
|
|
}
|
|
manifestOne := mustArtifact(t, mustNodeExecution(t, completedOne, "source"), "source-manifest.json")
|
|
manifestOnePayload := mustArtifactPayload(t, manifestOne)
|
|
if manifestOnePayload["newCount"] != float64(2) {
|
|
t.Fatalf("expected first run to emit full batch-a directory, got %#v", manifestOnePayload["newCount"])
|
|
}
|
|
if manifestOnePayload["incrementalUnit"] != "DIRECTORY" {
|
|
t.Fatalf("expected directory incremental unit, got %#v", manifestOnePayload["incrementalUnit"])
|
|
}
|
|
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "DJI_001", "clip-b.mp4"), "b")
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-b", "meta.json"), `{"project":"B"}`)
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-b", "DJI_001", "clip-c.mp4"), "c")
|
|
runTwo, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
|
|
if err != nil {
|
|
t.Fatalf("create second run: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process second run: %v", err)
|
|
}
|
|
|
|
completedTwo, err := dataStore.GetRun(ctx, runTwo.ID)
|
|
if err != nil {
|
|
t.Fatalf("get second run: %v", err)
|
|
}
|
|
manifestTwo := mustArtifact(t, mustNodeExecution(t, completedTwo, "source"), "source-manifest.json")
|
|
manifestTwoPayload := mustArtifactPayload(t, manifestTwo)
|
|
if manifestTwoPayload["listedCount"] != float64(5) {
|
|
t.Fatalf("expected listedCount=5 on second run, got %#v", manifestTwoPayload["listedCount"])
|
|
}
|
|
if manifestTwoPayload["newCount"] != float64(2) {
|
|
t.Fatalf("expected newCount=2 on second run, got %#v", manifestTwoPayload["newCount"])
|
|
}
|
|
entries, ok := manifestTwoPayload["entries"].([]any)
|
|
if !ok || len(entries) != 2 {
|
|
t.Fatalf("expected two emitted entries for new directory, got %#v", manifestTwoPayload["entries"])
|
|
}
|
|
for _, rawEntry := range entries {
|
|
entry, ok := rawEntry.(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("unexpected entry payload: %#v", rawEntry)
|
|
}
|
|
if !strings.Contains(fmt.Sprint(entry["path"]), "batch-b/") {
|
|
t.Fatalf("expected second run to emit only newly discovered directory contents, got %#v", entry["path"])
|
|
}
|
|
}
|
|
|
|
state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestTwoPayload["sourceFingerprint"].(string)))
|
|
if err != nil {
|
|
t.Fatalf("get source state after second run: %v", err)
|
|
}
|
|
if state == nil || len(state.SeenFingerprints) != 2 {
|
|
t.Fatalf("expected two seen directory keys after second run, got %#v", state)
|
|
}
|
|
if state.IncrementalUnit != "DIRECTORY" || state.DirectoryDepth != 1 {
|
|
t.Fatalf("expected committed directory incremental state, got %#v", state)
|
|
}
|
|
})
|
|
|
|
t.Run("does not commit state when downstream run fails", func(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "失败增量项目", Description: "incremental-fail"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, filepath.Join(sourceRoot, "batch-a", "meta.json"), `{"project":"A"}`)
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "增量失败工作流",
|
|
Description: "incremental fail",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "LOCAL"},
|
|
{Key: "SOURCE_PATH", Value: sourceRoot},
|
|
{Key: "SCAN_MODE", Value: "INCREMENTAL"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "fail",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "fail",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "echo boom >&2; exit 1"}},
|
|
},
|
|
Position: domain.Position{X: 240, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-fail", Source: "source", SourceHandle: "out-1", Target: "fail", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process failed run: %v", err)
|
|
}
|
|
|
|
failedRun, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get failed run: %v", err)
|
|
}
|
|
if failedRun == nil || failedRun.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %#v", failedRun)
|
|
}
|
|
manifest := mustArtifact(t, mustNodeExecution(t, failedRun, "source"), "source-manifest.json")
|
|
manifestPayload := mustArtifactPayload(t, manifest)
|
|
state, err := dataStore.GetSourceState(ctx, workflow.ID, "source", string(manifestPayload["sourceFingerprint"].(string)))
|
|
if err != nil {
|
|
t.Fatalf("get source state after failed run: %v", err)
|
|
}
|
|
if state != nil {
|
|
t.Fatalf("expected no committed source state after failed run, got %#v", state)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPrepareExecutionNodeResolvesStorageConnectionSecrets(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "BOS 主连接",
|
|
Provider: domain.StorageProviderBOS,
|
|
Endpoint: "https://gz.bcebos.com",
|
|
AccessKeyID: "ak-runtime-1234",
|
|
SecretAccessKey: "sk-runtime-5678",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
workflow := &domain.WorkflowRecord{
|
|
ID: "workflow-scan",
|
|
Kind: domain.WorkflowKindScan,
|
|
ScanConfig: &domain.ScanConfig{
|
|
DataSource: "BOS",
|
|
ConnectionID: connection.ID,
|
|
Bucket: "datastorage",
|
|
Prefix: "algo/X-platform-test",
|
|
ScanMode: "INCREMENTAL",
|
|
},
|
|
}
|
|
node := domain.WorkflowNode{
|
|
ID: "node-bos-scan-runtime",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "BOS 增量扫描",
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "BOS"},
|
|
},
|
|
},
|
|
}
|
|
|
|
prepared, err := runtime.prepareExecutionNode(ctx, workflow, node)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution node: %v", err)
|
|
}
|
|
params := nodeHyperparamsMap(prepared)
|
|
if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" {
|
|
t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"])
|
|
}
|
|
if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-5678" {
|
|
t.Fatalf("expected resolved bos secrets, got %#v", params)
|
|
}
|
|
if params["SOURCE_PATH"] != "bos://datastorage/algo/X-platform-test" {
|
|
t.Fatalf("expected source path from bucket/prefix, got %q", params["SOURCE_PATH"])
|
|
}
|
|
}
|
|
|
|
func TestPrepareExecutionNodeUsesRuntimeGlobalBOSEndpointForConnectionWithoutEndpoint(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "BOS 全局 endpoint 连接",
|
|
Provider: domain.StorageProviderBOS,
|
|
AccessKeyID: "ak-runtime-default",
|
|
SecretAccessKey: "sk-runtime-default",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
runtime.globalBOSEndpoint = "https://gz.bcebos.com"
|
|
|
|
workflow := &domain.WorkflowRecord{
|
|
ID: "workflow-default-bos",
|
|
Kind: domain.WorkflowKindScan,
|
|
ScanConfig: &domain.ScanConfig{
|
|
DataSource: "BOS",
|
|
ConnectionID: connection.ID,
|
|
Bucket: "datastorage",
|
|
Prefix: "algo/default-endpoint",
|
|
ScanMode: "INCREMENTAL",
|
|
},
|
|
}
|
|
node := domain.WorkflowNode{
|
|
ID: "node-default-bos",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "BOS 增量扫描",
|
|
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "BOS"}},
|
|
},
|
|
}
|
|
|
|
prepared, err := runtime.prepareExecutionNode(ctx, workflow, node)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution node: %v", err)
|
|
}
|
|
params := nodeHyperparamsMap(prepared)
|
|
if params["BOS_ENDPOINT"] != "https://gz.bcebos.com" {
|
|
t.Fatalf("expected runtime global bos endpoint, got %q", params["BOS_ENDPOINT"])
|
|
}
|
|
if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-default" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-default" {
|
|
t.Fatalf("expected resolved bos secrets, got %#v", params)
|
|
}
|
|
}
|
|
|
|
func TestPrepareExecutionNodeKeepsEmptyEndpointWithoutGlobalBOSFallback(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "BOS 无 endpoint 连接",
|
|
Provider: domain.StorageProviderBOS,
|
|
AccessKeyID: "ak-runtime-empty-endpoint",
|
|
SecretAccessKey: "sk-runtime-empty-endpoint",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
workflow := &domain.WorkflowRecord{
|
|
ID: "workflow-empty-bos-endpoint",
|
|
Kind: domain.WorkflowKindScan,
|
|
ScanConfig: &domain.ScanConfig{
|
|
DataSource: "BOS",
|
|
ConnectionID: connection.ID,
|
|
Bucket: "datastorage",
|
|
Prefix: "algo/empty-endpoint",
|
|
ScanMode: "INCREMENTAL",
|
|
},
|
|
}
|
|
node := domain.WorkflowNode{
|
|
ID: "node-empty-bos-endpoint",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "BOS 增量扫描",
|
|
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "BOS"}},
|
|
},
|
|
}
|
|
|
|
prepared, err := runtime.prepareExecutionNode(ctx, workflow, node)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution node: %v", err)
|
|
}
|
|
params := nodeHyperparamsMap(prepared)
|
|
if params["BOS_ENDPOINT"] != "" {
|
|
t.Fatalf("expected empty endpoint without global fallback, got %q", params["BOS_ENDPOINT"])
|
|
}
|
|
if params["BOS_ACCESS_KEY_ID"] != "ak-runtime-empty-endpoint" || params["BOS_SECRET_ACCESS_KEY"] != "sk-runtime-empty-endpoint" {
|
|
t.Fatalf("expected resolved bos secrets, got %#v", params)
|
|
}
|
|
}
|
|
|
|
func TestPrepareExecutionNodeResolvesNodeLevelStorageConnectionSecrets(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "BOS 节点连接",
|
|
Provider: domain.StorageProviderBOS,
|
|
Endpoint: "https://su.bcebos.com",
|
|
AccessKeyID: "ak-node-1234",
|
|
SecretAccessKey: "sk-node-5678",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
node := domain.WorkflowNode{
|
|
ID: "node-oss-source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "OSS 数据源",
|
|
ConnectionID: connection.ID,
|
|
Bucket: "datastorage",
|
|
Prefix: "factory/output",
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "SOURCE_PATH", Value: ""},
|
|
{Key: "DATA_SOURCE", Value: "BOS"},
|
|
},
|
|
},
|
|
}
|
|
|
|
prepared, err := runtime.prepareExecutionNode(ctx, nil, node)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution node: %v", err)
|
|
}
|
|
params := nodeHyperparamsMap(prepared)
|
|
if params["DATA_SOURCE"] != "BOS" {
|
|
t.Fatalf("expected BOS data source, got %q", params["DATA_SOURCE"])
|
|
}
|
|
if params["BOS_ENDPOINT"] != "https://su.bcebos.com" {
|
|
t.Fatalf("expected resolved endpoint, got %q", params["BOS_ENDPOINT"])
|
|
}
|
|
if params["BOS_ACCESS_KEY_ID"] != "ak-node-1234" || params["BOS_SECRET_ACCESS_KEY"] != "sk-node-5678" {
|
|
t.Fatalf("expected resolved node bos secrets, got %#v", params)
|
|
}
|
|
if params["SOURCE_PATH"] != "bos://datastorage/factory/output" {
|
|
t.Fatalf("expected source path from node bucket/prefix, got %q", params["SOURCE_PATH"])
|
|
}
|
|
}
|
|
|
|
func TestPrepareExecutionNodeResolvesMongoConnectionSecrets(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
connection, err := dataStore.CreateMongoConnection(ctx, domain.CreateMongoConnectionInput{
|
|
Name: "业务 MongoDB",
|
|
URI: "mongodb://business-mongo:27017",
|
|
Database: "business_data",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create mongo connection: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
node := domain.WorkflowNode{
|
|
ID: "node-mongo-write",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "MongoDB 写入",
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "MONGO_CONNECTION_ID", Value: connection.ID},
|
|
{Key: "MONGO_COLLECTION", Value: "qualityReports"},
|
|
},
|
|
},
|
|
}
|
|
|
|
prepared, err := runtime.prepareExecutionNode(ctx, nil, node)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution node: %v", err)
|
|
}
|
|
params := nodeHyperparamsMap(prepared)
|
|
if params["MONGO_URI"] != "mongodb://business-mongo:27017" {
|
|
t.Fatalf("expected resolved mongo uri, got %q", params["MONGO_URI"])
|
|
}
|
|
if params["MONGO_DB"] != "business_data" {
|
|
t.Fatalf("expected resolved mongo db, got %q", params["MONGO_DB"])
|
|
}
|
|
if params["MONGO_TARGET"] != "business" {
|
|
t.Fatalf("expected mongo target business, got %q", params["MONGO_TARGET"])
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableDispatchesUpToGlobalLimit(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 2
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Dispatch 项目", Description: "dispatch"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
createWorkflow := func(name string) *domain.WorkflowRecord {
|
|
t.Helper()
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: name,
|
|
Description: "dispatch test",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow %s: %v", name, err)
|
|
}
|
|
return workflow
|
|
}
|
|
|
|
workflowA := createWorkflow("并发工作流 A")
|
|
workflowB := createWorkflow("并发工作流 B")
|
|
runA, err := dataStore.CreateRun(ctx, workflowA.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run A: %v", err)
|
|
}
|
|
runB, err := dataStore.CreateRun(ctx, workflowB.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run B: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
runs, err := dataStore.ListRuns(ctx, "")
|
|
if err != nil {
|
|
t.Fatalf("list runs: %v", err)
|
|
}
|
|
runningCount := 0
|
|
for _, run := range runs {
|
|
if run.Status == domain.RunStatusRunning {
|
|
runningCount++
|
|
}
|
|
}
|
|
if runningCount != 2 {
|
|
t.Fatalf("expected 2 running runs after dispatch, got %d", runningCount)
|
|
}
|
|
|
|
waitForRunStatus(t, ctx, dataStore, runA.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
waitForRunStatus(t, ctx, dataStore, runB.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
}
|
|
|
|
func TestSelectQueuedRunsDoesNotLetAuditWaitsExhaustGlobalDispatch(t *testing.T) {
|
|
settings := &domain.SettingsRecord{}
|
|
settings.Worker.MaxConcurrentRuns = 2
|
|
workflows := []domain.WorkflowRecord{
|
|
{ID: "wf-human", MaxConcurrentRuns: 1},
|
|
{ID: "wf-data"},
|
|
{ID: "wf-video"},
|
|
}
|
|
runs := []domain.RunRecord{
|
|
{ID: "audit-wait", WorkflowID: "wf-human", Status: domain.RunStatusWaitingForAudit},
|
|
{ID: "human-next", WorkflowID: "wf-human", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:01Z"},
|
|
{ID: "data-next", WorkflowID: "wf-data", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:02Z"},
|
|
{ID: "video-next", WorkflowID: "wf-video", Status: domain.RunStatusQueued, CreatedAt: "2026-06-28T00:00:03Z"},
|
|
}
|
|
|
|
selected, queueDepth, activeTotal := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
|
|
|
|
if activeTotal != 0 {
|
|
t.Fatalf("waiting_for_audit should not consume global dispatch slots, activeTotal=%d", activeTotal)
|
|
}
|
|
if queueDepth != 3 {
|
|
t.Fatalf("queueDepth = %d, want 3", queueDepth)
|
|
}
|
|
if len(selected) != 2 {
|
|
t.Fatalf("selected = %#v, want two runs", selected)
|
|
}
|
|
if selected[0] == "human-next" || selected[1] == "human-next" {
|
|
t.Fatalf("same workflow audit wait should still block duplicate human run: %#v", selected)
|
|
}
|
|
if selected[0] != "data-next" || selected[1] != "video-next" {
|
|
t.Fatalf("selected = %#v, want data and video runs", selected)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableRespectsWorkflowConcurrencyOverride(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 3
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Override 项目", Description: "override"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
limitedWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "受限工作流",
|
|
Description: "limited",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxConcurrentRuns: 1,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create limited workflow: %v", err)
|
|
}
|
|
freeWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "自由工作流",
|
|
Description: "free",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create free workflow: %v", err)
|
|
}
|
|
|
|
runLimitedA, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create limited run A: %v", err)
|
|
}
|
|
runLimitedB, err := dataStore.CreateRun(ctx, limitedWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create limited run B: %v", err)
|
|
}
|
|
runFree, err := dataStore.CreateRun(ctx, freeWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create free run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
limitedRunAState, err := dataStore.GetRun(ctx, runLimitedA.ID)
|
|
if err != nil {
|
|
t.Fatalf("get limited run A: %v", err)
|
|
}
|
|
limitedRunBState, err := dataStore.GetRun(ctx, runLimitedB.ID)
|
|
if err != nil {
|
|
t.Fatalf("get limited run B: %v", err)
|
|
}
|
|
freeRunState, err := dataStore.GetRun(ctx, runFree.ID)
|
|
if err != nil {
|
|
t.Fatalf("get free run: %v", err)
|
|
}
|
|
if limitedRunAState.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected first limited run to be running, got %s", limitedRunAState.Status)
|
|
}
|
|
if limitedRunBState.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected second limited run to remain queued, got %s", limitedRunBState.Status)
|
|
}
|
|
if freeRunState.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected free workflow run to be running, got %s", freeRunState.Status)
|
|
}
|
|
|
|
waitForRunStatus(t, ctx, dataStore, runLimitedA.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
waitForRunStatus(t, ctx, dataStore, runFree.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available second pass: %v", err)
|
|
}
|
|
waitForRunStatus(t, ctx, dataStore, runLimitedB.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableSkipsStreamingWorkflowWithMissingProject(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "有效项目", Description: "valid"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
validWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "正常工作流",
|
|
Description: "valid workflow",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "command",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "command",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create valid workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, validWorkflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create queued run: %v", err)
|
|
}
|
|
|
|
orphanWorkflow := domain.WorkflowRecord{
|
|
ID: "workflow-orphan-streaming",
|
|
ProjectID: "project-missing",
|
|
Name: "孤儿增量工作流",
|
|
Description: "orphan streaming workflow",
|
|
Type: domain.WorkflowTypeStreaming,
|
|
Schedule: &domain.WorkflowSchedule{Interval: 30},
|
|
Nodes: []domain.WorkflowNode{},
|
|
Edges: []domain.WorkflowEdge{},
|
|
Status: domain.WorkflowStatusDraft,
|
|
CreatedAt: nowISO(),
|
|
UpdatedAt: nowISO(),
|
|
}
|
|
if err := dataStore.UpsertWorkflow(ctx, orphanWorkflow); err != nil {
|
|
t.Fatalf("upsert orphan workflow: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available should skip orphan workflow instead of failing: %v", err)
|
|
}
|
|
|
|
waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
}
|
|
|
|
func TestShouldEnqueueScheduledRunEveryHoursUsesAnchorTime(t *testing.T) {
|
|
workflow := domain.WorkflowRecord{
|
|
ID: "workflow-hours",
|
|
Type: domain.WorkflowTypeStreaming,
|
|
Schedule: &domain.WorkflowSchedule{
|
|
Mode: domain.WorkflowScheduleModeEveryHours,
|
|
IntervalHours: 4,
|
|
AnchorAt: "2026-04-21T10:15:00Z",
|
|
},
|
|
}
|
|
|
|
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 15, 0, 0, time.UTC)) {
|
|
t.Fatalf("expected first hourly window to enqueue at anchor time")
|
|
}
|
|
|
|
workflow.LastRunAt = "2026-04-21T10:15:00Z"
|
|
if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 14, 59, 0, time.UTC)) {
|
|
t.Fatalf("expected hourly schedule to wait until full interval elapsed")
|
|
}
|
|
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 14, 15, 0, 0, time.UTC)) {
|
|
t.Fatalf("expected hourly schedule to enqueue on the next anchored slot")
|
|
}
|
|
}
|
|
|
|
func TestShouldEnqueueScheduledRunEveryDaysAtFixedTimeUsesAnchorDate(t *testing.T) {
|
|
workflow := domain.WorkflowRecord{
|
|
ID: "workflow-days",
|
|
Type: domain.WorkflowTypeStreaming,
|
|
Schedule: &domain.WorkflowSchedule{
|
|
Mode: domain.WorkflowScheduleModeEveryDaysAtTime,
|
|
DayInterval: 2,
|
|
TimeOfDay: "18:30",
|
|
Timezone: "Asia/Shanghai",
|
|
AnchorAt: "2026-04-21T09:00:00Z",
|
|
},
|
|
}
|
|
|
|
if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 29, 59, 0, time.UTC)) {
|
|
t.Fatalf("expected daily schedule to wait until configured local time")
|
|
}
|
|
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 21, 10, 30, 0, 0, time.UTC)) {
|
|
t.Fatalf("expected daily schedule to enqueue at configured local time")
|
|
}
|
|
|
|
workflow.LastRunAt = "2026-04-21T10:30:00Z"
|
|
if shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 29, 59, 0, time.UTC)) {
|
|
t.Fatalf("expected next two-day slot to remain pending before local HH:mm")
|
|
}
|
|
if !shouldEnqueueScheduledRun(workflow, time.Date(2026, time.April, 23, 10, 30, 0, 0, time.UTC)) {
|
|
t.Fatalf("expected next two-day slot to enqueue at configured local HH:mm")
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunResumesQueuedPartialRunFromCompletedDependencies(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "恢复工作流",
|
|
Description: "resume",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}},
|
|
},
|
|
},
|
|
{
|
|
ID: "transform",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "JSON 映射重命名",
|
|
Category: "DATA",
|
|
Description: "transform",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "FROM_KEY", Value: "items"}, {Key: "TO_KEY", Value: "entries"}},
|
|
},
|
|
},
|
|
{
|
|
ID: "upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "sink",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}},
|
|
},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-transform", Source: "source", SourceHandle: "out-1", Target: "transform", TargetHandle: "in-1"},
|
|
{ID: "transform-upload", Source: "transform", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
run.Status = domain.RunStatusQueued
|
|
run.CurrentNodeID = ""
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "source",
|
|
NodeLabel: "真实数据源",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
StartedAt: nowISO(),
|
|
EndedAt: nowISO(),
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-source", RunID: run.ID, NodeID: "source", PortID: "out-1", Name: "payload.json", Type: domain.PortTypeJSON, Path: filepath.Join(t.TempDir(), "payload.json"), Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1}}},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if execution := mustNodeExecution(t, completed, "transform"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected transform to complete after resume, got %s", execution.Status)
|
|
}
|
|
if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected upload to complete after resume, got %s", execution.Status)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileActiveRunsRequeuesClaimedRunWithoutJobs(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "僵尸运行恢复",
|
|
Description: "recover",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
run.Status = domain.RunStatusRunning
|
|
run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano)
|
|
run.UpdatedAt = run.StartedAt
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("reconcile active runs: %v", err)
|
|
}
|
|
|
|
recovered, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get recovered run: %v", err)
|
|
}
|
|
if recovered == nil || recovered.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected run to be re-queued, got %#v", recovered)
|
|
}
|
|
if recovered.StartedAt != "" {
|
|
t.Fatalf("expected startedAt cleared after requeue, got %q", recovered.StartedAt)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileActiveRunsClearsQueuedRunWithMissingActiveJob(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover queued active"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "队列运行节点恢复",
|
|
Description: "recover queued active",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"ok\":true}"}}}}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
startedAt := time.Now().UTC().Add(-10 * time.Minute).Format(time.RFC3339Nano)
|
|
run.Status = domain.RunStatusQueued
|
|
run.CurrentNodeID = "source"
|
|
run.StartedAt = startedAt
|
|
run.UpdatedAt = startedAt
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{{
|
|
NodeID: "source",
|
|
NodeLabel: "真实数据源",
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: startedAt,
|
|
Logs: []domain.LogEntryRecord{{Timestamp: startedAt, Level: "INFO", Message: "Starting node execution", Operator: "真实数据源"}},
|
|
}}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("reconcile active runs: %v", err)
|
|
}
|
|
|
|
recovered, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get recovered run: %v", err)
|
|
}
|
|
if recovered == nil || recovered.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected run to remain queued, got %#v", recovered)
|
|
}
|
|
if recovered.CurrentNodeID != "" {
|
|
t.Fatalf("expected current node cleared, got %q", recovered.CurrentNodeID)
|
|
}
|
|
if len(recovered.NodeExecutions) != 1 || recovered.NodeExecutions[0].Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected stale source execution retained as failed recovery evidence, got %#v", recovered.NodeExecutions)
|
|
}
|
|
if !strings.Contains(recovered.NodeExecutions[0].Error, "recovery attempt 1/3") {
|
|
t.Fatalf("expected recovery attempt evidence, got %#v", recovered.NodeExecutions[0])
|
|
}
|
|
// Recovery deliberately applies a short backoff in production. Make this
|
|
// fixture dispatchable now so the test can verify the resumed execution.
|
|
recovered.AvailableAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *recovered); err != nil {
|
|
t.Fatalf("make recovered run dispatchable: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once after recovery: %v", err)
|
|
}
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if execution := mustNodeExecution(t, completed, "source"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected source to complete after stale execution recovery, got %s", execution.Status)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunResumesConnectedPendingNodes(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "resume connected"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "source-a", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"a\"}"}}}},
|
|
{ID: "source-b", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"b\"}"}}}},
|
|
{ID: "source-c", Type: "operator", Data: domain.WorkflowNodeData{Label: "真实数据源", Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}}, Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"name\":\"c\"}"}}}},
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "无边批次恢复",
|
|
Description: "resume connected",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: nodes,
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-a-to-source-b", Source: "source-a", SourceHandle: "out-1", Target: "source-b", TargetHandle: "in-1"},
|
|
{ID: "source-b-to-source-c", Source: "source-b", SourceHandle: "out-1", Target: "source-c", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
startedAt := nowISO()
|
|
run.Status = domain.RunStatusQueued
|
|
run.CurrentNodeID = ""
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{{
|
|
NodeID: "source-a",
|
|
NodeLabel: "真实数据源",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
StartedAt: startedAt,
|
|
EndedAt: startedAt,
|
|
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-a", RunID: run.ID, NodeID: "source-a", PortID: "out-1", Name: "a.json", Type: domain.PortTypeJSON, Data: map[string]any{"name": "a"}}},
|
|
}}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if len(completed.NodeExecutions) != 3 {
|
|
t.Fatalf("expected all unconnected nodes executed, got %#v", completed.NodeExecutions)
|
|
}
|
|
for _, node := range nodes {
|
|
if execution := mustNodeExecution(t, completed, node.ID); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected %s completed, got %s", node.ID, execution.Status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileActiveRunsRecoversSucceededNodeAndResumesDownstream(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复项目", Description: "recover success"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "成功节点恢复",
|
|
Description: "recover success",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "DATA_SOURCE", Value: "INLINE"}, {Key: "SOURCE_TEXT", Value: "{\"items\":[{\"name\":\"sample\"}],\"count\":1}"}},
|
|
},
|
|
},
|
|
{
|
|
ID: "annotate",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "视频标注",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "annotations", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{{Key: "MODEL", Value: "stub"}},
|
|
},
|
|
},
|
|
{
|
|
ID: "upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "printf '{\"ok\":true}' > done.json"}},
|
|
},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-annotate", Source: "source", SourceHandle: "out-1", Target: "annotate", TargetHandle: "in-1"},
|
|
{ID: "annotate-upload", Source: "annotate", SourceHandle: "out-1", Target: "upload", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
sourceArtifact := domain.ExecutionArtifact{
|
|
ID: "artifact-source",
|
|
RunID: run.ID,
|
|
NodeID: "source",
|
|
PortID: "out-1",
|
|
Name: "payload.json",
|
|
Type: domain.PortTypeJSON,
|
|
Path: filepath.Join(t.TempDir(), "payload.json"),
|
|
Data: map[string]any{"items": []any{map[string]any{"name": "sample"}}, "count": 1},
|
|
}
|
|
run.Status = domain.RunStatusRunning
|
|
run.CurrentNodeID = "annotate"
|
|
run.StartedAt = time.Now().UTC().Add(-2 * time.Minute).Format(time.RFC3339Nano)
|
|
run.UpdatedAt = run.StartedAt
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "source",
|
|
NodeLabel: "真实数据源",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
StartedAt: run.StartedAt,
|
|
EndedAt: run.StartedAt,
|
|
Logs: []domain.LogEntryRecord{},
|
|
Artifacts: []domain.ExecutionArtifact{sourceArtifact},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
},
|
|
{
|
|
NodeID: "annotate",
|
|
NodeLabel: "视频标注",
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: run.StartedAt,
|
|
Logs: []domain.LogEntryRecord{{Timestamp: run.StartedAt, Level: "INFO", Message: "Starting node execution", Operator: "视频标注"}},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
DataRefs: []domain.DataRefRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
launcher := testLauncher{
|
|
launch: func(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
env := append([]string{}, os.Environ()...)
|
|
keys := make([]string, 0, len(input.Env))
|
|
for key := range input.Env {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
env = append(env, "RUN_ID="+input.RunID, "WORKFLOW_ID="+input.WorkflowID, "NODE_ID="+input.NodeID, "NODE_LABEL="+input.NodeLabel)
|
|
for _, key := range keys {
|
|
env = append(env, key+"="+input.Env[key])
|
|
}
|
|
cmd := exec.CommandContext(ctx, "python3", operatorRunnerPath(t))
|
|
cmd.Env = env
|
|
output, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: %s", err, string(output))
|
|
}
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: fmt.Sprintf("%s-%s", input.RunID, input.NodeID), Namespace: "droboticflow"},
|
|
Output: string(output),
|
|
}, nil
|
|
},
|
|
inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) {
|
|
if runID == run.ID && nodeID == "annotate" {
|
|
return &k8s.JobInspection{
|
|
Reference: &k8s.JobReference{JobName: "job-annotate", Namespace: "droboticflow"},
|
|
State: k8s.JobStateSucceeded,
|
|
Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"annotation-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"annotate\",\"portId\":\"out-1\",\"name\":\"annotation.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/annotation.json\",\"data\":{\"items\":[{\"label\":\"walking\"}],\"count\":1}}],\"logs\":[],\"resourceSeries\":[]}",
|
|
}, nil
|
|
}
|
|
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
|
|
},
|
|
}
|
|
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, launcher)
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("reconcile active runs: %v", err)
|
|
}
|
|
recovered, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get recovered run: %v", err)
|
|
}
|
|
if recovered == nil || recovered.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected recovered run queued, got %#v", recovered)
|
|
}
|
|
if execution := mustNodeExecution(t, recovered, "annotate"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected annotate completed after recovery, got %s", execution.Status)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once after recovery: %v", err)
|
|
}
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if execution := mustNodeExecution(t, completed, "upload"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected downstream upload completed, got %s", execution.Status)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileActiveRunsCompletesRunWhenRecoveredFinalNodeSucceeded(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "恢复完成项目", Description: "recover final success"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "最终节点恢复",
|
|
Description: "recover final success",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
|
|
},
|
|
},
|
|
{
|
|
ID: "sample",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "抽检任务",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "samples", Type: domain.PortTypeJSON}},
|
|
},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-sample", Source: "source", SourceHandle: "out-1", Target: "sample", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
startedAt := time.Now().UTC().Add(-3 * time.Minute).Format(time.RFC3339Nano)
|
|
run.Status = domain.RunStatusRunning
|
|
run.CurrentNodeID = "sample"
|
|
run.StartedAt = startedAt
|
|
run.UpdatedAt = startedAt
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "source",
|
|
NodeLabel: "真实数据源",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
StartedAt: startedAt,
|
|
EndedAt: startedAt,
|
|
Artifacts: []domain.ExecutionArtifact{{
|
|
ID: "artifact-source",
|
|
RunID: run.ID,
|
|
NodeID: "source",
|
|
PortID: "out-1",
|
|
Name: "payload.json",
|
|
Type: domain.PortTypeJSON,
|
|
Data: map[string]any{"ok": true},
|
|
}},
|
|
},
|
|
{
|
|
NodeID: "sample",
|
|
NodeLabel: "抽检任务",
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: startedAt,
|
|
Logs: []domain.LogEntryRecord{{Timestamp: startedAt, Level: "INFO", Message: "Starting node execution", Operator: "抽检任务"}},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{
|
|
inspect: func(ctx context.Context, runID string, nodeID string) (*k8s.JobInspection, error) {
|
|
if runID == run.ID && nodeID == "sample" {
|
|
return &k8s.JobInspection{
|
|
Reference: &k8s.JobReference{JobName: "job-sample", Namespace: "droboticflow"},
|
|
State: k8s.JobStateSucceeded,
|
|
Output: "DROBOTICFLOW_RESULT={\"artifacts\":[{\"id\":\"sample-1\",\"runId\":\"" + run.ID + "\",\"nodeId\":\"sample\",\"portId\":\"out-1\",\"name\":\"audit-samples.json\",\"type\":\"json\",\"size\":\"1 KB\",\"path\":\"/tmp/audit-samples.json\",\"data\":{\"sampledCount\":1}}],\"logs\":[],\"resourceSeries\":[]}",
|
|
}, nil
|
|
}
|
|
return &k8s.JobInspection{State: k8s.JobStateMissing}, nil
|
|
},
|
|
})
|
|
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("reconcile active runs: %v", err)
|
|
}
|
|
completed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get completed run: %v", err)
|
|
}
|
|
if completed == nil || completed.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected recovered final node to complete run, got %#v", completed)
|
|
}
|
|
if completed.CurrentNodeID != "" {
|
|
t.Fatalf("expected current node cleared, got %q", completed.CurrentNodeID)
|
|
}
|
|
if execution := mustNodeExecution(t, completed, "sample"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected sample completed, got %s", execution.Status)
|
|
}
|
|
finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
|
|
if err != nil {
|
|
t.Fatalf("get workflow: %v", err)
|
|
}
|
|
if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted {
|
|
t.Fatalf("expected workflow completed, got %#v", finalWorkflow)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunExecutesIndependentNodesInParallel(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 3
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Parallel DAG 项目", Description: "parallel dag"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "并行分支工作流",
|
|
Description: "parallel dag",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxConcurrentRuns: 3,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "OSS 数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "payload", Type: domain.PortTypeJSON}},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "INLINE"},
|
|
{Key: "SOURCE_TEXT", Value: "parallel"},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
{
|
|
ID: "branch-a",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "branch a",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-a", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"a\"}' > result-a.json"}},
|
|
},
|
|
Position: domain.Position{X: 200, Y: -120},
|
|
},
|
|
{
|
|
ID: "branch-b",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "branch b",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-b", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"b\"}' > result-b.json"}},
|
|
},
|
|
Position: domain.Position{X: 200, Y: 0},
|
|
},
|
|
{
|
|
ID: "branch-c",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "branch c",
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result-c", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 2 && printf '{\"branch\":\"c\"}' > result-c.json"}},
|
|
},
|
|
Position: domain.Position{X: 200, Y: 120},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "source-a", Source: "source", SourceHandle: "out-1", Target: "branch-a", TargetHandle: "in-1"},
|
|
{ID: "source-b", Source: "source", SourceHandle: "out-1", Target: "branch-b", TargetHandle: "in-1"},
|
|
{ID: "source-c", Source: "source", SourceHandle: "out-1", Target: "branch-c", TargetHandle: "in-1"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
runningRun := waitForRunningNodeCount(t, ctx, dataStore, run.ID, 3, 5*time.Second)
|
|
if runningRun.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected run to be running while branches execute, got %s", runningRun.Status)
|
|
}
|
|
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second)
|
|
for _, nodeID := range []string{"branch-a", "branch-b", "branch-c"} {
|
|
if execution := mustNodeExecution(t, completed, nodeID); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected node %s to complete, got %s", nodeID, execution.Status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunContinuesConnectedBranchesPastConcurrencyLimit(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.MaxConcurrentRuns = 2
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Disconnected 项目", Description: "disconnected nodes"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
nodes := []domain.WorkflowNode{{
|
|
ID: "root",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "root node",
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"node":"root"}' > root.json`}},
|
|
},
|
|
}}
|
|
edges := make([]domain.WorkflowEdge, 0, 5)
|
|
for index := 0; index < 5; index++ {
|
|
nodeID := fmt.Sprintf("independent-%d", index)
|
|
nodes = append(nodes, domain.WorkflowNode{
|
|
ID: nodeID,
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "independent node",
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: fmt.Sprintf(`printf '{"node":"%s"}' > %s.json`, nodeID, nodeID)}},
|
|
},
|
|
})
|
|
edges = append(edges, domain.WorkflowEdge{ID: "root-to-" + nodeID, Source: "root", Target: nodeID})
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "单连通分支工作流",
|
|
Description: "connected branch batch",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxConcurrentRuns: 2,
|
|
Nodes: nodes,
|
|
Edges: edges,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 10*time.Second)
|
|
if len(completed.NodeExecutions) != len(nodes) {
|
|
t.Fatalf("expected %d executed nodes, got %d: %#v", len(nodes), len(completed.NodeExecutions), completed.NodeExecutions)
|
|
}
|
|
for _, node := range nodes {
|
|
if execution := mustNodeExecution(t, completed, node.ID); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected node %s to complete, got %s", node.ID, execution.Status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeRerunSubtreeOnlyExecutesSelectedSubgraph(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "子链路重跑项目", Description: "rerun subtree"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "子链路重跑工作流",
|
|
Description: "rerun subtree",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "root",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "FLOW",
|
|
Description: "root",
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"root":true}' > root.json`}},
|
|
},
|
|
},
|
|
{
|
|
ID: "start-a",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "FLOW",
|
|
Description: "branch a start",
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a"}' > branch-a.json`}},
|
|
},
|
|
},
|
|
{
|
|
ID: "downstream-a",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "FLOW",
|
|
Description: "branch a end",
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"a-downstream"}' > branch-a-downstream.json`}},
|
|
},
|
|
},
|
|
{
|
|
ID: "start-b",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "FLOW",
|
|
Description: "branch b start",
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: `printf '{"branch":"b"}' > branch-b.json`}},
|
|
},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{
|
|
{ID: "root-to-start-a", Source: "root", Target: "start-a"},
|
|
{ID: "root-to-start-b", Source: "root", Target: "start-b"},
|
|
{ID: "a-to-downstream", Source: "start-a", Target: "downstream-a"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
run.Trigger = domain.RunTriggerRerunSubtree
|
|
run.RestartFromNodeID = "start-a"
|
|
startedAt := nowISO()
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{{
|
|
NodeID: "root",
|
|
NodeLabel: "Shell 命令执行",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
StartedAt: startedAt,
|
|
EndedAt: startedAt,
|
|
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-root", RunID: run.ID, NodeID: "root", PortID: "out-1", Name: "root.json", Type: domain.PortTypeJSON, Data: map[string]any{"root": true}}},
|
|
}}
|
|
run.Status = domain.RunStatusQueued
|
|
run.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert rerun subtree run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if execution := mustNodeExecution(t, completed, "start-a"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected start-a to complete, got %s", execution.Status)
|
|
}
|
|
if execution := mustNodeExecution(t, completed, "downstream-a"); execution.Status != domain.NodeExecutionStatusCompleted {
|
|
t.Fatalf("expected downstream-a to complete, got %s", execution.Status)
|
|
}
|
|
for _, execution := range completed.NodeExecutions {
|
|
if execution.NodeID == "start-b" {
|
|
t.Fatalf("expected unrelated branch start-b not to execute, got %#v", execution)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessRunKeepsClaimedRunRunningWhenExistingNodeExecutionIsActive(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "runner:latest",
|
|
workspaceRoot: "/workspace",
|
|
launcher: testLauncher{},
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: "",
|
|
Name: "resume-active-node",
|
|
Description: "resume active execution",
|
|
Nodes: []domain.WorkflowNode{{ID: "source", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source"}}},
|
|
Edges: []domain.WorkflowEdge{},
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
Schedule: nil,
|
|
ScanConfig: nil,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
run.Status = domain.RunStatusQueued
|
|
run.CurrentNodeID = "source"
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "source",
|
|
NodeLabel: "Source",
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: nowISO(),
|
|
},
|
|
}
|
|
run.UpdatedAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
processed := mustProcessOnceRun(t, runtime, ctx)
|
|
if processed.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected running status, got %s", processed.Status)
|
|
}
|
|
if processed.CurrentNodeID != "source" {
|
|
t.Fatalf("expected currentNodeID=source, got %s", processed.CurrentNodeID)
|
|
}
|
|
if execution := mustNodeExecution(t, processed, "source"); execution.Status != domain.NodeExecutionStatusRunning {
|
|
t.Fatalf("expected source execution running, got %s", execution.Status)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunFailsWhenWorkflowTimeoutReached(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Timeout 项目", Description: "timeout"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "超时工作流",
|
|
Description: "timeout",
|
|
Type: domain.WorkflowTypeBatch,
|
|
RunTimeoutSeconds: 1,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "slow-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "slow",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
failed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get failed run: %v", err)
|
|
}
|
|
if failed == nil || failed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %#v", failed)
|
|
}
|
|
if !strings.Contains(failed.Error, "超时") {
|
|
t.Fatalf("expected timeout error, got %q", failed.Error)
|
|
}
|
|
execution := mustNodeExecution(t, failed, "slow-node")
|
|
if execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed node execution, got %s", execution.Status)
|
|
}
|
|
|
|
storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
|
|
if err != nil {
|
|
t.Fatalf("get workflow: %v", err)
|
|
}
|
|
if storedWorkflow == nil || storedWorkflow.Status != domain.WorkflowStatusFailed {
|
|
t.Fatalf("expected failed workflow, got %#v", storedWorkflow)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunAutoRetriesFailedWorkflows(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
markerPath := filepath.Join(t.TempDir(), "retry-marker")
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry 项目", Description: "retry"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "自动重试工作流",
|
|
Description: "retry",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxRetryAttempts: 1,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "flaky-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "flaky",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{
|
|
Key: "COMMAND",
|
|
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
|
|
}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create initial run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("first process once: %v", err)
|
|
}
|
|
|
|
firstAttempt, err := dataStore.GetRun(ctx, initialRun.ID)
|
|
if err != nil {
|
|
t.Fatalf("get first attempt: %v", err)
|
|
}
|
|
if firstAttempt == nil || firstAttempt.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected first attempt to fail, got %#v", firstAttempt)
|
|
}
|
|
if firstAttempt.Attempt != 1 {
|
|
t.Fatalf("expected first attempt number to be 1, got %d", firstAttempt.Attempt)
|
|
}
|
|
|
|
runs, err := dataStore.ListRuns(ctx, "")
|
|
if err != nil {
|
|
t.Fatalf("list runs: %v", err)
|
|
}
|
|
if len(runs) != 2 {
|
|
t.Fatalf("expected original + retry run, got %d", len(runs))
|
|
}
|
|
|
|
var retryRun domain.RunRecord
|
|
foundRetry := false
|
|
for _, candidate := range runs {
|
|
if candidate.ID == initialRun.ID {
|
|
continue
|
|
}
|
|
retryRun = candidate
|
|
foundRetry = true
|
|
break
|
|
}
|
|
if !foundRetry {
|
|
t.Fatalf("expected retry run to be created")
|
|
}
|
|
if retryRun.Trigger != domain.RunTriggerRetry {
|
|
t.Fatalf("expected retry trigger, got %s", retryRun.Trigger)
|
|
}
|
|
if retryRun.Attempt != 2 {
|
|
t.Fatalf("expected retry attempt number to be 2, got %d", retryRun.Attempt)
|
|
}
|
|
if retryRun.ParentRunID != initialRun.ID {
|
|
t.Fatalf("expected retry parent run %s, got %s", initialRun.ID, retryRun.ParentRunID)
|
|
}
|
|
if retryRun.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected retry run to be queued, got %s", retryRun.Status)
|
|
}
|
|
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("second process once: %v", err)
|
|
}
|
|
|
|
completedRetry, err := dataStore.GetRun(ctx, retryRun.ID)
|
|
if err != nil {
|
|
t.Fatalf("get retry run: %v", err)
|
|
}
|
|
if completedRetry == nil || completedRetry.Status != domain.RunStatusCompleted {
|
|
t.Fatalf("expected retry run to complete, got %#v", completedRetry)
|
|
}
|
|
|
|
finalWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
|
|
if err != nil {
|
|
t.Fatalf("get final workflow: %v", err)
|
|
}
|
|
if finalWorkflow == nil || finalWorkflow.Status != domain.WorkflowStatusCompleted {
|
|
t.Fatalf("expected workflow to recover and complete, got %#v", finalWorkflow)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessRunFailsWhenNodeTimeoutReached(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Node Timeout 项目", Description: "node timeout"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "节点超时工作流",
|
|
Description: "node timeout",
|
|
Type: domain.WorkflowTypeBatch,
|
|
NodeTimeoutSeconds: 1,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "slow-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "slow",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{Key: "COMMAND", Value: "sleep 3 && printf '{\"ok\":true}' > result.json"}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("process once: %v", err)
|
|
}
|
|
|
|
failed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get failed run: %v", err)
|
|
}
|
|
if failed == nil || failed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected failed run, got %#v", failed)
|
|
}
|
|
if !strings.Contains(failed.Error, "节点执行超时") {
|
|
t.Fatalf("expected node timeout error, got %q", failed.Error)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
markerPath := filepath.Join(t.TempDir(), "retry-backoff-marker")
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Retry Backoff 项目", Description: "retry backoff"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "重试延迟工作流",
|
|
Description: "retry backoff",
|
|
Type: domain.WorkflowTypeBatch,
|
|
MaxRetryAttempts: 1,
|
|
RetryBackoffSeconds: 2,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "flaky-node",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Shell 命令执行",
|
|
Category: "UTIL",
|
|
Description: "flaky",
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "result", Type: domain.PortTypeFile}},
|
|
Hyperparams: []domain.HyperParam{{
|
|
Key: "COMMAND",
|
|
Value: "if [ ! -f '" + markerPath + "' ]; then touch '" + markerPath + "' && exit 1; fi; printf '{\"ok\":true}' > result.json",
|
|
}},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
initialRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create initial run: %v", err)
|
|
}
|
|
|
|
runtime, err := newTestRuntime(t, dataStore)
|
|
if err != nil {
|
|
t.Fatalf("create runtime: %v", err)
|
|
}
|
|
if _, err := runtime.ProcessOnce(ctx); err != nil {
|
|
t.Fatalf("first process once: %v", err)
|
|
}
|
|
|
|
runs, err := dataStore.ListRuns(ctx, "")
|
|
if err != nil {
|
|
t.Fatalf("list runs: %v", err)
|
|
}
|
|
if len(runs) != 2 {
|
|
t.Fatalf("expected original + retry run, got %d", len(runs))
|
|
}
|
|
var retryRun domain.RunRecord
|
|
for _, candidate := range runs {
|
|
if candidate.ID != initialRun.ID {
|
|
retryRun = candidate
|
|
break
|
|
}
|
|
}
|
|
if retryRun.AvailableAt == "" {
|
|
t.Fatalf("expected retry run to carry availableAt")
|
|
}
|
|
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available before backoff expires: %v", err)
|
|
}
|
|
stillQueued, err := dataStore.GetRun(ctx, retryRun.ID)
|
|
if err != nil {
|
|
t.Fatalf("get retry run before backoff: %v", err)
|
|
}
|
|
if stillQueued == nil || stillQueued.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected retry run to remain queued before backoff expires, got %#v", stillQueued)
|
|
}
|
|
|
|
time.Sleep(2200 * time.Millisecond)
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available after backoff expires: %v", err)
|
|
}
|
|
completedRetry := waitForRunStatus(t, ctx, dataStore, retryRun.ID, domain.RunStatusCompleted, 8*time.Second)
|
|
if completedRetry.Attempt != 2 {
|
|
t.Fatalf("expected retry attempt number to stay 2, got %d", completedRetry.Attempt)
|
|
}
|
|
}
|
|
|
|
func newReconcileFixture(t *testing.T, dataStore store.Store, name string) (*domain.WorkflowRecord, *domain.RunRecord) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: name + " project", Description: "reconcile"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: name + " workflow",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{
|
|
ID: "node-1", Type: "operator", Data: domain.WorkflowNodeData{Label: name + " node"},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
startedAt := time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano)
|
|
run.Status = domain.RunStatusRunning
|
|
run.CurrentNodeID = "node-1"
|
|
run.StartedAt = startedAt
|
|
run.UpdatedAt = startedAt
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{{
|
|
NodeID: "node-1", NodeLabel: name + " node", Attempt: 1, Status: domain.NodeExecutionStatusRunning, StartedAt: startedAt,
|
|
}}
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
return workflow, run
|
|
}
|
|
|
|
func TestRuntimeReconcileFailedJobTerminalizesRunAndNode(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
workflow, run := newReconcileFixture(t, dataStore, "failed job")
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) {
|
|
return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "container exited with code 2"}, nil
|
|
}})
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("reconcile active runs: %v", err)
|
|
}
|
|
failed, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "code 2") {
|
|
t.Fatalf("expected failed run with diagnostics, got %#v", failed)
|
|
}
|
|
if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_failed" || failed.Failure.NodeID != "node-1" {
|
|
t.Fatalf("expected structured kubernetes failure, got %#v", failed.Failure)
|
|
}
|
|
if execution := mustNodeExecution(t, failed, "node-1"); execution.Status != domain.NodeExecutionStatusFailed {
|
|
t.Fatalf("expected failed node execution, got %#v", execution)
|
|
}
|
|
storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID)
|
|
if err != nil || storedWorkflow.Status != domain.WorkflowStatusFailed {
|
|
t.Fatalf("expected failed workflow, got %#v err=%v", storedWorkflow, err)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileStuckImagePullTerminalizesButHealthyLongJobStaysRunning(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
_, blockedRun := newReconcileFixture(t, dataStore, "blocked image")
|
|
_, healthyRun := newReconcileFixture(t, dataStore, "healthy training")
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) {
|
|
if runID == blockedRun.ID {
|
|
return &k8s.JobInspection{
|
|
State: k8s.JobStatePending, CreatedAt: time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano),
|
|
BlockingReason: "ImagePullBackOff", BlockingMessage: "back-off pulling image",
|
|
}, nil
|
|
}
|
|
return &k8s.JobInspection{State: k8s.JobStateActive, CreatedAt: time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339Nano)}, nil
|
|
}})
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("reconcile active runs: %v", err)
|
|
}
|
|
blocked, _ := dataStore.GetRun(ctx, blockedRun.ID)
|
|
healthy, _ := dataStore.GetRun(ctx, healthyRun.ID)
|
|
if blocked.Status != domain.RunStatusFailed || !strings.Contains(blocked.Error, "ImagePullBackOff") {
|
|
t.Fatalf("expected blocked image run to fail explicitly, got %#v", blocked)
|
|
}
|
|
if blocked.Failure == nil || blocked.Failure.Code != "kubernetes_image_pull_blocked" || blocked.Failure.Stage != "scheduling" {
|
|
t.Fatalf("expected structured image pull failure, got %#v", blocked.Failure)
|
|
}
|
|
if healthy.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected healthy long-running job unchanged, got %#v", healthy)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileMissingJobFailsAfterRecoveryLimit(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
_, run := newReconcileFixture(t, dataStore, "missing job")
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
|
|
runtime.missingMaxRecoveries = 1
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("first reconcile: %v", err)
|
|
}
|
|
recovered, _ := dataStore.GetRun(ctx, run.ID)
|
|
if recovered.Status != domain.RunStatusQueued || !strings.Contains(mustNodeExecution(t, recovered, "node-1").Error, "recovery attempt 1/1") {
|
|
t.Fatalf("expected first recovery to requeue, got %#v", recovered)
|
|
}
|
|
recovered.Status = domain.RunStatusRunning
|
|
recovered.CurrentNodeID = "node-1"
|
|
recovered.UpdatedAt = time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano)
|
|
execution := mustNodeExecution(t, recovered, "node-1")
|
|
execution.Status = domain.NodeExecutionStatusRunning
|
|
execution.StartedAt = recovered.UpdatedAt
|
|
recovered.NodeExecutions = []domain.NodeExecutionRecord{execution}
|
|
if err := dataStore.UpsertRun(ctx, *recovered); err != nil {
|
|
t.Fatalf("simulate recovered execution: %v", err)
|
|
}
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("second reconcile: %v", err)
|
|
}
|
|
failed, _ := dataStore.GetRun(ctx, run.ID)
|
|
if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "after 1 recovery attempts") {
|
|
t.Fatalf("expected missing job to fail after recovery limit, got %#v", failed)
|
|
}
|
|
if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_missing" {
|
|
t.Fatalf("expected structured missing-job failure, got %#v", failed.Failure)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeReconcileIsolatesPerRunInspectionErrors(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
_, brokenRun := newReconcileFixture(t, dataStore, "inspection error")
|
|
_, failedRun := newReconcileFixture(t, dataStore, "independent failure")
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) {
|
|
if runID == brokenRun.ID {
|
|
return nil, fmt.Errorf("temporary kubernetes error")
|
|
}
|
|
return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "independent failure"}, nil
|
|
}})
|
|
if err := runtime.reconcileActiveRuns(ctx); err != nil {
|
|
t.Fatalf("per-run error must not fail the cycle: %v", err)
|
|
}
|
|
broken, _ := dataStore.GetRun(ctx, brokenRun.ID)
|
|
failed, _ := dataStore.GetRun(ctx, failedRun.ID)
|
|
if broken.Status != domain.RunStatusRunning {
|
|
t.Fatalf("expected errored inspection unchanged, got %#v", broken)
|
|
}
|
|
if failed.Status != domain.RunStatusFailed {
|
|
t.Fatalf("expected independent run reconciled, got %#v", failed)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeLeaseOwnerChangeCancelsExecutionContext(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
_, run := newReconcileFixture(t, dataStore, "lease loss")
|
|
run.ExecutionLeaseOwner = "other-worker"
|
|
run.ExecutionLeaseExpiresAt = time.Now().UTC().Add(time.Hour).Format(time.RFC3339Nano)
|
|
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
|
t.Fatalf("upsert foreign lease: %v", err)
|
|
}
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
|
|
runtime.dispatchOwner = "this-worker"
|
|
runtime.runLeaseTTL = 30 * time.Millisecond
|
|
leaseCtx, cancelLease := context.WithCancelCause(ctx)
|
|
defer cancelLease(context.Canceled)
|
|
go runtime.renewRunLeaseUntil(leaseCtx, run.ID, cancelLease)
|
|
deadline := time.After(time.Second)
|
|
for !errors.Is(context.Cause(leaseCtx), errRunLeaseLost) {
|
|
select {
|
|
case <-deadline:
|
|
t.Fatalf("expected lease owner change to cancel execution context, cause=%v", context.Cause(leaseCtx))
|
|
case <-time.After(5 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRuntimeProcessAvailableAppliesPersistedPollInterval(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
settings.Worker.PollIntervalMS = 750
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("upsert settings: %v", err)
|
|
}
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{})
|
|
runtime.SetConfiguredPollInterval(2 * time.Second)
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
t.Fatalf("process available: %v", err)
|
|
}
|
|
if got := runtime.EffectivePollInterval(); got != 750*time.Millisecond {
|
|
t.Fatalf("expected persisted 750ms poll interval, got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestLatestStructuredProgressUsesLatestValidMarker(t *testing.T) {
|
|
logs := strings.Join([]string{
|
|
"ordinary training output",
|
|
`DROBOTICFLOW_PROGRESS {"stage":"train","epoch":1,"step":20,"maxStep":100,"samplesPerSecond":4.5,"estimatedRemainingSeconds":18,"checkpoint":"checkpoint-20","updatedAt":"2026-07-10T08:00:00Z"}`,
|
|
"DROBOTICFLOW_PROGRESS {partial",
|
|
`prefix DROBOTICFLOW_PROGRESS {"stage":"train","epoch":2,"step":40,"maxStep":100,"samplesPerSecond":5.5,"estimatedRemainingSeconds":11,"checkpoint":"checkpoint-40","updatedAt":"2026-07-10T08:01:00Z"}`,
|
|
}, "\n")
|
|
progress, err := latestStructuredProgress(logs)
|
|
if err != nil {
|
|
t.Fatalf("parse progress: %v", err)
|
|
}
|
|
if progress == nil || progress.Epoch != 2 || progress.Step != 40 || progress.Checkpoint != "checkpoint-40" {
|
|
t.Fatalf("expected latest valid marker, got %#v", progress)
|
|
}
|
|
}
|
|
|
|
func TestNodeProgressReporterPersistsStructuredMarker(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
_, run := newReconcileFixture(t, dataStore, "structured progress")
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{tailNodePodLogs: func(context.Context, string, string, int64) (string, error) {
|
|
return `DROBOTICFLOW_PROGRESS {"stage":"train","epoch":3,"step":75,"maxStep":100,"samplesPerSecond":6.25,"estimatedRemainingSeconds":4,"checkpoint":"checkpoint-75"}`, nil
|
|
}})
|
|
reporterCtx, cancelReporter := context.WithCancel(ctx)
|
|
done := make(chan struct{})
|
|
go runtime.runNodeProgressReporterWithInterval(reporterCtx, run.ID, "node-1", false, 5*time.Millisecond, func() { close(done) })
|
|
deadline := time.Now().Add(time.Second)
|
|
for {
|
|
stored, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if stored.Progress != nil {
|
|
if stored.Progress.Step != 75 || stored.Progress.Checkpoint != "checkpoint-75" || stored.Progress.UpdatedAt == "" {
|
|
t.Fatalf("unexpected structured progress: %#v", stored.Progress)
|
|
}
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("timed out waiting for structured progress")
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
cancelReporter()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("progress reporter did not stop")
|
|
}
|
|
}
|
|
|
|
func TestSyncStopMetadataPreservesAtomicProgressUpdates(t *testing.T) {
|
|
target := &domain.RunRecord{}
|
|
source := &domain.RunRecord{Progress: &domain.RunProgress{Stage: "train", Step: 9, UpdatedAt: "2026-07-10T08:00:00Z"}}
|
|
syncStopMetadata(target, source)
|
|
if target.Progress == nil || target.Progress.Step != 9 {
|
|
t.Fatalf("expected progress copied from persisted run, got %#v", target.Progress)
|
|
}
|
|
source.Progress.Step = 10
|
|
if target.Progress.Step != 9 {
|
|
t.Fatalf("expected progress copy to avoid pointer aliasing, got %#v", target.Progress)
|
|
}
|
|
}
|
|
|
|
func TestCompletedNodePersistsTerminalStructuredProgressMarker(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "terminal progress", Description: "progress"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID, Name: "terminal progress", Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{{ID: "train", Type: "operator", Data: domain.WorkflowNodeData{Label: "train"}}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create run: %v", err)
|
|
}
|
|
runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(_ context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) {
|
|
stageManifestPath := input.Env["STAGE_MANIFEST_PATH"]
|
|
if err := os.MkdirAll(filepath.Dir(stageManifestPath), 0o755); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.WriteFile(stageManifestPath, []byte(`{"dataRefs":[]}`), 0o644); err != nil {
|
|
return nil, err
|
|
}
|
|
return &k8s.LaunchResult{
|
|
Reference: &k8s.JobReference{JobName: "terminal-progress", Namespace: "droboticflow"},
|
|
Output: strings.Join([]string{
|
|
`DROBOTICFLOW_PROGRESS {"stage":"completed","epoch":4,"step":100,"maxStep":100,"samplesPerSecond":6.5,"estimatedRemainingSeconds":0,"checkpoint":"checkpoint-final"}`,
|
|
`DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`,
|
|
}, "\n"),
|
|
}, nil
|
|
}})
|
|
result, err := runtime.ProcessOnce(ctx)
|
|
if err != nil {
|
|
t.Fatalf("process run: %v", err)
|
|
}
|
|
if result == nil || result.Run == nil || result.Run.ID != run.ID {
|
|
t.Fatalf("unexpected process result: %#v", result)
|
|
}
|
|
completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, time.Second)
|
|
if completed.Progress == nil || completed.Progress.Stage != "completed" || completed.Progress.Step != 100 || completed.Progress.Checkpoint != "checkpoint-final" {
|
|
t.Fatalf("expected terminal structured progress, got %#v", completed.Progress)
|
|
}
|
|
}
|
|
|
|
// shouldReportNodeProgress must flag only custom-container operators — builtin
|
|
// operator-runner pods self-report resourceSeries via their in-process ResourceSampler,
|
|
// so platform-side GPU sampling is reserved for nodes whose resolved image differs from
|
|
// the operator-runner image (mirroring k8s.Launch's shouldSample gate).
|
|
func TestShouldReportNodeProgressOnlyForCustomContainers(t *testing.T) {
|
|
rt := &Runtime{operatorRunnerImage: "droboticflow-operator-runner:latest"}
|
|
cases := []struct {
|
|
name string
|
|
node domain.WorkflowNode
|
|
want bool
|
|
}{
|
|
{"custom image", domain.WorkflowNode{Data: domain.WorkflowNodeData{Image: "sam3-video-tracker:latest"}}, true},
|
|
{"builtin operator-runner image", domain.WorkflowNode{Data: domain.WorkflowNodeData{Image: "droboticflow-operator-runner:latest"}}, false},
|
|
{"digest-pinned custom image", domain.WorkflowNode{Data: domain.WorkflowNodeData{OperatorImageID: "sam3-video-tracker@sha256:40efc6ba77e0bfb3a029825b20a36615445f7c62a0279804971c543c46ccd08f"}}, true},
|
|
{"no image falls back to operator-runner", domain.WorkflowNode{Data: domain.WorkflowNodeData{}}, false},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := shouldReportNodeProgress(rt, tc.node); got != tc.want {
|
|
t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// mapGPUSample must round-trip every GPU-relevant field the launcher reports onto the
|
|
// resource monitor's domain.ResourceSample (the same mapping the executor uses when
|
|
// folding the launcher's final series).
|
|
func TestMapGPUSampleCopiesFields(t *testing.T) {
|
|
in := k8s.GPUSample{
|
|
Timestamp: "2026-07-10T00:00:00.000000Z",
|
|
GPU: 42,
|
|
GPUMemoryUsedMiB: 28000,
|
|
GPUMemoryTotalMiB: 81920,
|
|
GPUName: "NVIDIA A800-SXM4-80GB",
|
|
GPUCount: 1,
|
|
}
|
|
out := mapGPUSample(in)
|
|
if out.Timestamp != in.Timestamp || out.GPU != in.GPU || out.GPUMemoryUsedMiB != in.GPUMemoryUsedMiB ||
|
|
out.GPUMemoryTotalMiB != in.GPUMemoryTotalMiB || out.GPUName != in.GPUName || out.GPUCount != in.GPUCount {
|
|
t.Fatalf("mapGPUSample mismatch: in=%#v out=%#v", in, out)
|
|
}
|
|
}
|