droboticflow/internal/worker/executor_test.go

523 lines
19 KiB
Go

package worker
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"droboticflow/internal/domain"
)
func TestBuildNodeEnvInjectsPlatformMongoEnvForMongoNode(t *testing.T) {
t.Setenv("MONGO_URI", "mongodb://platform-mongo:27017")
t.Setenv("MONGO_DB", "droboticflow")
run := &domain.RunRecord{
ID: "run-1",
ParentRunID: "run-parent",
WorkflowID: "workflow-1",
Trigger: domain.RunTriggerScheduled,
PreviousRunAt: "2026-04-21T10:00:00Z",
CurrentRunAt: "2026-04-21T11:30:00Z",
}
workflow := &domain.WorkflowRecord{
ID: "workflow-1",
Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "from-workflow"}},
}
node := domain.WorkflowNode{
ID: "node-1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "MongoDB 写入",
Category: "DATA",
Description: "write docs",
Hyperparams: []domain.HyperParam{
{Key: "MONGO_COLLECTION", Value: "qualityReports"},
{Key: "PARENT_RUN_ID", Value: "user-value-must-not-win"},
},
},
}
workspace := nodeWorkspace{
RunRoot: "/tmp/workspace/system/runs/run-1",
Root: "/tmp/workspace/system/runs/run-1/nodes/node-1",
OutputDir: "/tmp/workspace/system/runs/run-1/nodes/node-1/outputs",
InputArtifactsPath: "/tmp/workspace/system/runs/run-1/nodes/node-1/input-artifacts.json",
ResultPath: "/tmp/workspace/system/runs/run-1/nodes/node-1/result.json",
StageManifestPath: "/tmp/workspace/system/runs/run-1/nodes/node-1/stage-manifest.json",
}
env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
if env["PLATFORM_MONGO_URI"] != "mongodb://platform-mongo:27017" {
t.Fatalf("expected PLATFORM_MONGO_URI to be injected, got %q", env["PLATFORM_MONGO_URI"])
}
if env["PLATFORM_MONGO_DB"] != "droboticflow" {
t.Fatalf("expected PLATFORM_MONGO_DB to be injected, got %q", env["PLATFORM_MONGO_DB"])
}
if _, ok := env["BUSINESS_MONGO_URI"]; ok {
t.Fatalf("did not expect BUSINESS_MONGO_URI to be injected")
}
if _, ok := env["BUSINESS_MONGO_DB"]; ok {
t.Fatalf("did not expect BUSINESS_MONGO_DB to be injected")
}
if env["MONGO_COLLECTION"] != "qualityReports" {
t.Fatalf("expected node hyperparam preserved, got %q", env["MONGO_COLLECTION"])
}
if env["GLOBAL_ENV"] != "from-workflow" {
t.Fatalf("expected workflow hyperparam injected, got %q", env["GLOBAL_ENV"])
}
if env["INPUT_ARTIFACTS_PATH"] != workspace.InputArtifactsPath {
t.Fatalf("expected INPUT_ARTIFACTS_PATH to be populated, got %q", env["INPUT_ARTIFACTS_PATH"])
}
if env["RESULT_OUTPUT_PATH"] != workspace.ResultPath {
t.Fatalf("expected RESULT_OUTPUT_PATH to be populated, got %q", env["RESULT_OUTPUT_PATH"])
}
if env["STAGE_MANIFEST_PATH"] != workspace.StageManifestPath {
t.Fatalf("expected STAGE_MANIFEST_PATH to be populated, got %q", env["STAGE_MANIFEST_PATH"])
}
if env["WORKSPACE_ROOT"] != workspace.RunRoot {
t.Fatalf("expected WORKSPACE_ROOT to point at run root, got %q", env["WORKSPACE_ROOT"])
}
if env["PARENT_RUN_ID"] != "run-parent" {
t.Fatalf("expected protected parent run id, got %q", env["PARENT_RUN_ID"])
}
if env["PARENT_RUN_WORKSPACE_ROOT"] != "/tmp/workspace/system/runs/run-parent" {
t.Fatalf("expected parent run workspace root, got %q", env["PARENT_RUN_WORKSPACE_ROOT"])
}
if env["PARENT_NODE_OUTPUT_DIR"] != "/tmp/workspace/system/runs/run-parent/nodes/node-1/outputs" {
t.Fatalf("expected parent node output directory, got %q", env["PARENT_NODE_OUTPUT_DIR"])
}
expectedPrevious := "2026-04-21T18:00:00+08:00"
if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != expectedPrevious {
t.Fatalf("expected previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"])
}
expectedCurrent := "2026-04-21T19:30:00+08:00"
if env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] != expectedCurrent {
t.Fatalf("expected current run timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"])
}
}
func TestRuntimeContractLongRunning(t *testing.T) {
if runtimeContractLongRunning(nil) {
t.Fatalf("nil runtime contract should not be long-running")
}
if runtimeContractLongRunning(map[string]any{"longRunning": false}) {
t.Fatalf("false runtime contract should not be long-running")
}
if !runtimeContractLongRunning(map[string]any{"longRunning": true}) {
t.Fatalf("true runtime contract should be long-running")
}
if runtimeContractLongRunning(map[string]any{"longRunning": "true"}) {
t.Fatalf("non-boolean runtime contract should not be long-running")
}
}
func TestBuildNodeEnvWritesInputArtifactsManifest(t *testing.T) {
run := &domain.RunRecord{
ID: "run-1",
WorkflowID: "workflow-1",
Trigger: domain.RunTriggerManual,
CurrentRunAt: "2026-04-21T11:30:00Z",
}
workflow := &domain.WorkflowRecord{
ID: "workflow-1",
Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "workflow-default"}},
}
node := domain.WorkflowNode{
ID: "node-1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "数据过滤",
Category: "DATA",
Description: "filter docs",
Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "node-override"}},
},
}
root := t.TempDir()
workspace := nodeWorkspace{
RunRoot: filepath.Join(root, "system", "runs", "run-1"),
Root: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1"),
OutputDir: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "outputs"),
InputArtifactsPath: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "input-artifacts.json"),
ResultPath: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "result.json"),
StageManifestPath: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "stage-manifest.json"),
}
inputs := []domain.ExecutionArtifact{{ID: "artifact-1", Name: "manifest.json", Type: domain.PortTypeJSON, Path: "bos://bucket/demo.json"}}
env, err := buildNodeEnv(run, workflow, node, inputs, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
payload, err := os.ReadFile(workspace.InputArtifactsPath)
if err != nil {
t.Fatalf("read input manifest: %v", err)
}
var decoded []domain.ExecutionArtifact
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("decode input manifest: %v", err)
}
if len(decoded) != 1 || decoded[0].ID != "artifact-1" {
t.Fatalf("unexpected input manifest payload: %#v", decoded)
}
if env["INPUT_ARTIFACTS_JSON"] == "" {
t.Fatalf("expected small input manifest to remain inline for convenience")
}
if _, ok := env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]; ok {
t.Fatalf("expected manual run to omit previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"])
}
if _, ok := env["WORKFLOW_CURRENT_RUN_TIMESTAMP"]; ok {
t.Fatalf("expected manual run to omit current run timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"])
}
if env["GLOBAL_ENV"] != "node-override" {
t.Fatalf("expected node hyperparams to override workflow-level env, got %q", env["GLOBAL_ENV"])
}
}
func TestGatherInputLineagePreservesWorkflowEdgeHandles(t *testing.T) {
workflow := domain.WorkflowRecord{
Edges: []domain.WorkflowEdge{
{ID: "edge-frames", Source: "source", SourceHandle: "frames", Target: "sink", TargetHandle: "video-input"},
{ID: "edge-summary", Source: "source", SourceHandle: "summary", Target: "sink", TargetHandle: "metadata-input"},
},
}
executionMap := map[string]domain.NodeExecutionRecord{
"source": {
NodeID: "source",
Artifacts: []domain.ExecutionArtifact{
{ID: "artifact-frames", NodeID: "source", PortID: "frames", PortName: "frames", Name: "frames.json", Type: domain.PortTypeJSON},
{ID: "artifact-summary", NodeID: "source", PortID: "summary", PortName: "summary", Name: "summary.json", Type: domain.PortTypeJSON},
},
},
}
lineage := gatherInputLineage(workflow, executionMap, "sink")
if len(lineage) != 2 {
t.Fatalf("expected 2 input lineage records, got %#v", lineage)
}
if lineage[0].WorkflowEdgeID != "edge-frames" || lineage[0].SourceHandle != "frames" || lineage[0].TargetHandle != "video-input" || lineage[0].Artifact.ID != "artifact-frames" {
t.Fatalf("unexpected first lineage record: %#v", lineage[0])
}
if lineage[1].WorkflowEdgeID != "edge-summary" || lineage[1].SourceHandle != "summary" || lineage[1].TargetHandle != "metadata-input" || lineage[1].Artifact.ID != "artifact-summary" {
t.Fatalf("unexpected second lineage record: %#v", lineage[1])
}
artifacts := inputArtifactsFromLineage(lineage)
if len(artifacts) != 2 || artifacts[0].ID != "artifact-frames" || artifacts[1].ID != "artifact-summary" {
t.Fatalf("unexpected artifacts from lineage: %#v", artifacts)
}
}
func TestBuildNodeEnvDoesNotAllowHyperparamsToOverrideRunTimestampEnv(t *testing.T) {
run := &domain.RunRecord{
ID: "run-2",
WorkflowID: "workflow-2",
Trigger: domain.RunTriggerScheduled,
PreviousRunAt: "2026-04-21T10:00:00Z",
CurrentRunAt: "2026-04-21T11:30:00Z",
}
workflow := &domain.WorkflowRecord{
ID: "workflow-2",
Hyperparams: []domain.HyperParam{
{Key: "WORKFLOW_PREVIOUS_RUN_TIMESTAMP", Value: "workflow-999"},
{Key: "WORKFLOW_CURRENT_RUN_TIMESTAMP", Value: "workflow-888"},
},
}
node := domain.WorkflowNode{
ID: "node-2",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Description: "exec",
Hyperparams: []domain.HyperParam{
{Key: "WORKFLOW_PREVIOUS_RUN_TIMESTAMP", Value: "999"},
{Key: "WORKFLOW_CURRENT_RUN_TIMESTAMP", Value: "888"},
},
},
}
workspace := nodeWorkspace{
RunRoot: "/tmp/workspace/system/runs/run-2",
Root: "/tmp/workspace/system/runs/run-2/nodes/node-2",
OutputDir: "/tmp/workspace/system/runs/run-2/nodes/node-2/outputs",
InputArtifactsPath: "/tmp/workspace/system/runs/run-2/nodes/node-2/input-artifacts.json",
ResultPath: "/tmp/workspace/system/runs/run-2/nodes/node-2/result.json",
StageManifestPath: "/tmp/workspace/system/runs/run-2/nodes/node-2/stage-manifest.json",
}
env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != "2026-04-21T18:00:00+08:00" {
t.Fatalf("expected protected previous timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"])
}
if env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] != "2026-04-21T19:30:00+08:00" {
t.Fatalf("expected protected current timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"])
}
}
func TestBuildNodeEnvInjectsProtectedOperatorCommandPolicy(t *testing.T) {
t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "false")
t.Setenv("OPERATOR_COMMAND_ALLOWLIST_REGEX", `^python3?\s`)
t.Setenv("OPERATOR_COMMAND_DENYLIST_REGEX", `(^|\s)rm\s+-rf`)
run := &domain.RunRecord{
ID: "run-3",
WorkflowID: "workflow-3",
Trigger: domain.RunTriggerManual,
}
workflow := &domain.WorkflowRecord{
ID: "workflow-3",
Hyperparams: []domain.HyperParam{
{Key: "OPERATOR_ALLOW_SHELL_COMMANDS", Value: "true"},
{Key: "OPERATOR_COMMAND_ALLOWLIST_REGEX", Value: ".*"},
},
}
node := domain.WorkflowNode{
ID: "node-3",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Hyperparams: []domain.HyperParam{
{Key: "OPERATOR_COMMAND_DENYLIST_REGEX", Value: ""},
},
},
}
workspace := nodeWorkspace{
RunRoot: "/tmp/workspace/system/runs/run-3",
Root: "/tmp/workspace/system/runs/run-3/nodes/node-3",
OutputDir: "/tmp/workspace/system/runs/run-3/nodes/node-3/outputs",
InputArtifactsPath: "/tmp/workspace/system/runs/run-3/nodes/node-3/input-artifacts.json",
ResultPath: "/tmp/workspace/system/runs/run-3/nodes/node-3/result.json",
StageManifestPath: "/tmp/workspace/system/runs/run-3/nodes/node-3/stage-manifest.json",
}
env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
if env["OPERATOR_ALLOW_SHELL_COMMANDS"] != "false" {
t.Fatalf("expected platform shell command flag to win, got %q", env["OPERATOR_ALLOW_SHELL_COMMANDS"])
}
if env["OPERATOR_COMMAND_ALLOWLIST_REGEX"] != `^python3?\s` {
t.Fatalf("expected platform allowlist to win, got %q", env["OPERATOR_COMMAND_ALLOWLIST_REGEX"])
}
if env["OPERATOR_COMMAND_DENYLIST_REGEX"] != `(^|\s)rm\s+-rf` {
t.Fatalf("expected platform denylist to win, got %q", env["OPERATOR_COMMAND_DENYLIST_REGEX"])
}
}
func TestParseExecutionOutputReadsResultFilePayload(t *testing.T) {
resultPath := filepath.Join(t.TempDir(), "result.json")
payload := executionResult{
Logs: []domain.LogEntryRecord{{Level: "INFO", Message: "done", Operator: "测试节点"}},
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-1", Name: "report.json"}},
}
data, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
if err := os.WriteFile(resultPath, data, 0o644); err != nil {
t.Fatalf("write payload: %v", err)
}
parsed, err := parseExecutionOutput("some log\n"+resultPathPrefix+resultPath+"\n", "测试节点")
if err != nil {
t.Fatalf("parseExecutionOutput returned error: %v", err)
}
if len(parsed.Artifacts) != 1 || parsed.Artifacts[0].ID != "artifact-1" {
t.Fatalf("unexpected parsed artifacts: %#v", parsed.Artifacts)
}
}
func TestParseExecutionOutputAcceptsNumericArtifactSize(t *testing.T) {
parsed, err := parseExecutionOutput(`DROBOTICFLOW_RESULT={"artifacts":[{"id":"artifact-size","name":"training_report","type":"file","size":1852,"path":"/workspace/report.json"}]}`, "训练")
if err != nil {
t.Fatalf("parseExecutionOutput returned error: %v", err)
}
if len(parsed.Artifacts) != 1 {
t.Fatalf("expected one artifact, got %#v", parsed.Artifacts)
}
artifact := parsed.Artifacts[0]
if artifact.SizeBytes != 1852 || artifact.Size != "1.8 KB" {
t.Fatalf("unexpected numeric artifact size: %#v", artifact)
}
}
func TestLoadStageManifestDataRefs(t *testing.T) {
stageManifestPath := filepath.Join(t.TempDir(), "stage-manifest.json")
payload := map[string]any{
"dataRefs": []map[string]any{
{
"artifactId": "artifact-1",
"artifactName": "dataset-index.json",
"kind": "dataset",
"datasetType": "x-platform-index",
"manifestPath": "/workspace/run/stage/manifest.json",
"payloadPath": "/workspace/run/stage/payload.json",
"partitionDir": "/workspace/run/stage/partitions",
"partitionCount": 3,
"itemCount": 12,
},
},
}
data, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
if err := os.WriteFile(stageManifestPath, data, 0o644); err != nil {
t.Fatalf("write stage manifest: %v", err)
}
dataRefs, err := loadStageManifestDataRefs(stageManifestPath)
if err != nil {
t.Fatalf("loadStageManifestDataRefs returned error: %v", err)
}
if len(dataRefs) != 1 || dataRefs[0].DatasetType != "x-platform-index" {
t.Fatalf("unexpected data refs: %#v", dataRefs)
}
}
func TestApplyNodeOverridesToNode(t *testing.T) {
node := domain.WorkflowNode{
ID: "test-node",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Original Operator",
Image: "registry.example.com/base:v1",
Hyperparams: []domain.HyperParam{
{Key: "threshold", Value: "0.5"},
{Key: "batch_size", Value: "16"},
},
ContainerCommand: []string{"python", "main.py"},
ContainerArgs: []string{"--mode", "normal"},
},
}
override := domain.NodeOverride{
Hyperparams: []domain.HyperParam{
{Key: "threshold", Value: "0.9"},
{Key: "learning_rate", Value: "0.001"},
},
Image: "registry.example.com/custom:v2",
ContainerCommand: []string{"python", "debug.py"},
ContainerArgs: []string{"--debug", "--verbose"},
}
overridden := applyNodeOverridesToNode(node, override)
if overridden.Data.Image != "registry.example.com/custom:v2" {
t.Fatalf("expected overridden image, got %s", overridden.Data.Image)
}
if len(overridden.Data.ContainerCommand) != 2 || overridden.Data.ContainerCommand[1] != "debug.py" {
t.Fatalf("expected overridden command, got %#v", overridden.Data.ContainerCommand)
}
if len(overridden.Data.ContainerArgs) != 2 || overridden.Data.ContainerArgs[0] != "--debug" {
t.Fatalf("expected overridden args, got %#v", overridden.Data.ContainerArgs)
}
hpMap := make(map[string]string)
for _, hp := range overridden.Data.Hyperparams {
hpMap[hp.Key] = hp.Value
}
if hpMap["threshold"] != "0.9" {
t.Fatalf("expected threshold=0.9, got %s", hpMap["threshold"])
}
if hpMap["batch_size"] != "16" {
t.Fatalf("expected batch_size=16, got %s", hpMap["batch_size"])
}
if hpMap["learning_rate"] != "0.001" {
t.Fatalf("expected learning_rate=0.001, got %s", hpMap["learning_rate"])
}
// Verify original node was NOT modified (zero mutation invariant)
if node.Data.Image != "registry.example.com/base:v1" {
t.Fatalf("original node image was corrupted")
}
if len(node.Data.Hyperparams) != 2 || node.Data.Hyperparams[0].Value != "0.5" {
t.Fatalf("original node hyperparams were corrupted")
}
}
func TestGatherNodeInputsWithNodeOverrides(t *testing.T) {
workflow := domain.WorkflowRecord{
ID: "wf-1",
Nodes: []domain.WorkflowNode{
{ID: "node-1", Data: domain.WorkflowNodeData{Label: "Node 1"}},
},
}
run := &domain.RunRecord{
ID: "run-dbg-1",
NodeOverrides: map[string]domain.NodeOverride{
"node-1": {
Inputs: map[string]domain.ArtifactPayload{
"in_port": {
Mode: domain.ArtifactTransferModeInline,
Type: domain.PortTypeJSON,
Inline: map[string]any{"debug_key": "debug_val"},
},
},
},
},
}
artifacts, lineage, bindings := gatherNodeInputs(workflow, run, map[string]domain.NodeExecutionRecord{}, "node-1")
_ = lineage
if len(bindings) != 1 {
t.Fatalf("expected 1 binding, got %d", len(bindings))
}
if bindings[0].TargetPortID != "in_port" {
t.Fatalf("expected target port in_port, got %s", bindings[0].TargetPortID)
}
if len(artifacts) != 1 {
t.Fatalf("expected 1 artifact, got %d", len(artifacts))
}
if artifacts[0].PortID != "in_port" {
t.Fatalf("expected port in_port, got %s", artifacts[0].PortID)
}
}
func TestRerunAllowedNodeSetSingleNodeDebug(t *testing.T) {
wf := domain.WorkflowRecord{
ID: "wf-dag",
Nodes: []domain.WorkflowNode{
{ID: "node-A"},
{ID: "node-B"},
{ID: "node-C"},
},
Edges: []domain.WorkflowEdge{
{ID: "e1", Source: "node-A", Target: "node-B"},
{ID: "e2", Source: "node-B", Target: "node-C"},
},
}
debugRun := &domain.RunRecord{
ID: "run-single-node",
Trigger: domain.RunTriggerManual,
Mode: domain.RunModeDebug,
DebugTarget: &domain.DebugExecutionTarget{
TargetNodeID: "node-B",
SingleNode: true,
},
}
allowed := rerunAllowedNodeSet(wf, debugRun)
if len(allowed) != 1 {
t.Fatalf("expected allowed set of size 1, got %d", len(allowed))
}
if _, ok := allowed["node-B"]; !ok {
t.Fatalf("expected node-B to be allowed")
}
if _, ok := allowed["node-A"]; ok {
t.Fatalf("did not expect node-A in single-node debug allowed set")
}
if _, ok := allowed["node-C"]; ok {
t.Fatalf("did not expect node-C in single-node debug allowed set")
}
}