droboticflow-new/internal/worker/executor_test.go
2026-07-11 00:11:36 +08:00

370 lines
14 KiB
Go

package worker
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"droboticflow/internal/domain"
)
func TestBuildNodeEnvInjectsPlatformMongoEnvForMongoNode(t *testing.T) {
t.Setenv("MONGO_URI", "mongodb://platform-mongo:27017")
t.Setenv("MONGO_DB", "droboticflow")
run := &domain.RunRecord{
ID: "run-1",
ParentRunID: "run-parent",
WorkflowID: "workflow-1",
Trigger: domain.RunTriggerScheduled,
PreviousRunAt: "2026-04-21T10:00:00Z",
CurrentRunAt: "2026-04-21T11:30:00Z",
}
workflow := &domain.WorkflowRecord{
ID: "workflow-1",
Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "from-workflow"}},
}
node := domain.WorkflowNode{
ID: "node-1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "MongoDB 写入",
Category: "DATA",
Description: "write docs",
Hyperparams: []domain.HyperParam{
{Key: "MONGO_COLLECTION", Value: "qualityReports"},
{Key: "PARENT_RUN_ID", Value: "user-value-must-not-win"},
},
},
}
workspace := nodeWorkspace{
RunRoot: "/tmp/workspace/system/runs/run-1",
Root: "/tmp/workspace/system/runs/run-1/nodes/node-1",
OutputDir: "/tmp/workspace/system/runs/run-1/nodes/node-1/outputs",
InputArtifactsPath: "/tmp/workspace/system/runs/run-1/nodes/node-1/input-artifacts.json",
ResultPath: "/tmp/workspace/system/runs/run-1/nodes/node-1/result.json",
StageManifestPath: "/tmp/workspace/system/runs/run-1/nodes/node-1/stage-manifest.json",
}
env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
if env["PLATFORM_MONGO_URI"] != "mongodb://platform-mongo:27017" {
t.Fatalf("expected PLATFORM_MONGO_URI to be injected, got %q", env["PLATFORM_MONGO_URI"])
}
if env["PLATFORM_MONGO_DB"] != "droboticflow" {
t.Fatalf("expected PLATFORM_MONGO_DB to be injected, got %q", env["PLATFORM_MONGO_DB"])
}
if _, ok := env["BUSINESS_MONGO_URI"]; ok {
t.Fatalf("did not expect BUSINESS_MONGO_URI to be injected")
}
if _, ok := env["BUSINESS_MONGO_DB"]; ok {
t.Fatalf("did not expect BUSINESS_MONGO_DB to be injected")
}
if env["MONGO_COLLECTION"] != "qualityReports" {
t.Fatalf("expected node hyperparam preserved, got %q", env["MONGO_COLLECTION"])
}
if env["GLOBAL_ENV"] != "from-workflow" {
t.Fatalf("expected workflow hyperparam injected, got %q", env["GLOBAL_ENV"])
}
if env["INPUT_ARTIFACTS_PATH"] != workspace.InputArtifactsPath {
t.Fatalf("expected INPUT_ARTIFACTS_PATH to be populated, got %q", env["INPUT_ARTIFACTS_PATH"])
}
if env["RESULT_OUTPUT_PATH"] != workspace.ResultPath {
t.Fatalf("expected RESULT_OUTPUT_PATH to be populated, got %q", env["RESULT_OUTPUT_PATH"])
}
if env["STAGE_MANIFEST_PATH"] != workspace.StageManifestPath {
t.Fatalf("expected STAGE_MANIFEST_PATH to be populated, got %q", env["STAGE_MANIFEST_PATH"])
}
if env["WORKSPACE_ROOT"] != workspace.RunRoot {
t.Fatalf("expected WORKSPACE_ROOT to point at run root, got %q", env["WORKSPACE_ROOT"])
}
if env["PARENT_RUN_ID"] != "run-parent" {
t.Fatalf("expected protected parent run id, got %q", env["PARENT_RUN_ID"])
}
if env["PARENT_RUN_WORKSPACE_ROOT"] != "/tmp/workspace/system/runs/run-parent" {
t.Fatalf("expected parent run workspace root, got %q", env["PARENT_RUN_WORKSPACE_ROOT"])
}
if env["PARENT_NODE_OUTPUT_DIR"] != "/tmp/workspace/system/runs/run-parent/nodes/node-1/outputs" {
t.Fatalf("expected parent node output directory, got %q", env["PARENT_NODE_OUTPUT_DIR"])
}
expectedPrevious := "2026-04-21T18:00:00+08:00"
if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != expectedPrevious {
t.Fatalf("expected previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"])
}
expectedCurrent := "2026-04-21T19:30:00+08:00"
if env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] != expectedCurrent {
t.Fatalf("expected current run timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"])
}
}
func TestRuntimeContractLongRunning(t *testing.T) {
if runtimeContractLongRunning(nil) {
t.Fatalf("nil runtime contract should not be long-running")
}
if runtimeContractLongRunning(map[string]any{"longRunning": false}) {
t.Fatalf("false runtime contract should not be long-running")
}
if !runtimeContractLongRunning(map[string]any{"longRunning": true}) {
t.Fatalf("true runtime contract should be long-running")
}
if runtimeContractLongRunning(map[string]any{"longRunning": "true"}) {
t.Fatalf("non-boolean runtime contract should not be long-running")
}
}
func TestBuildNodeEnvWritesInputArtifactsManifest(t *testing.T) {
run := &domain.RunRecord{
ID: "run-1",
WorkflowID: "workflow-1",
Trigger: domain.RunTriggerManual,
CurrentRunAt: "2026-04-21T11:30:00Z",
}
workflow := &domain.WorkflowRecord{
ID: "workflow-1",
Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "workflow-default"}},
}
node := domain.WorkflowNode{
ID: "node-1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "数据过滤",
Category: "DATA",
Description: "filter docs",
Hyperparams: []domain.HyperParam{{Key: "GLOBAL_ENV", Value: "node-override"}},
},
}
root := t.TempDir()
workspace := nodeWorkspace{
RunRoot: filepath.Join(root, "system", "runs", "run-1"),
Root: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1"),
OutputDir: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "outputs"),
InputArtifactsPath: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "input-artifacts.json"),
ResultPath: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "result.json"),
StageManifestPath: filepath.Join(root, "system", "runs", "run-1", "nodes", "node-1", "stage-manifest.json"),
}
inputs := []domain.ExecutionArtifact{{ID: "artifact-1", Name: "manifest.json", Type: domain.PortTypeJSON, Path: "bos://bucket/demo.json"}}
env, err := buildNodeEnv(run, workflow, node, inputs, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
payload, err := os.ReadFile(workspace.InputArtifactsPath)
if err != nil {
t.Fatalf("read input manifest: %v", err)
}
var decoded []domain.ExecutionArtifact
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("decode input manifest: %v", err)
}
if len(decoded) != 1 || decoded[0].ID != "artifact-1" {
t.Fatalf("unexpected input manifest payload: %#v", decoded)
}
if env["INPUT_ARTIFACTS_JSON"] == "" {
t.Fatalf("expected small input manifest to remain inline for convenience")
}
if _, ok := env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]; ok {
t.Fatalf("expected manual run to omit previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"])
}
if _, ok := env["WORKFLOW_CURRENT_RUN_TIMESTAMP"]; ok {
t.Fatalf("expected manual run to omit current run timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"])
}
if env["GLOBAL_ENV"] != "node-override" {
t.Fatalf("expected node hyperparams to override workflow-level env, got %q", env["GLOBAL_ENV"])
}
}
func TestGatherInputLineagePreservesWorkflowEdgeHandles(t *testing.T) {
workflow := domain.WorkflowRecord{
Edges: []domain.WorkflowEdge{
{ID: "edge-frames", Source: "source", SourceHandle: "frames", Target: "sink", TargetHandle: "video-input"},
{ID: "edge-summary", Source: "source", SourceHandle: "summary", Target: "sink", TargetHandle: "metadata-input"},
},
}
executionMap := map[string]domain.NodeExecutionRecord{
"source": {
NodeID: "source",
Artifacts: []domain.ExecutionArtifact{
{ID: "artifact-frames", NodeID: "source", PortID: "frames", PortName: "frames", Name: "frames.json", Type: domain.PortTypeJSON},
{ID: "artifact-summary", NodeID: "source", PortID: "summary", PortName: "summary", Name: "summary.json", Type: domain.PortTypeJSON},
},
},
}
lineage := gatherInputLineage(workflow, executionMap, "sink")
if len(lineage) != 2 {
t.Fatalf("expected 2 input lineage records, got %#v", lineage)
}
if lineage[0].WorkflowEdgeID != "edge-frames" || lineage[0].SourceHandle != "frames" || lineage[0].TargetHandle != "video-input" || lineage[0].Artifact.ID != "artifact-frames" {
t.Fatalf("unexpected first lineage record: %#v", lineage[0])
}
if lineage[1].WorkflowEdgeID != "edge-summary" || lineage[1].SourceHandle != "summary" || lineage[1].TargetHandle != "metadata-input" || lineage[1].Artifact.ID != "artifact-summary" {
t.Fatalf("unexpected second lineage record: %#v", lineage[1])
}
artifacts := inputArtifactsFromLineage(lineage)
if len(artifacts) != 2 || artifacts[0].ID != "artifact-frames" || artifacts[1].ID != "artifact-summary" {
t.Fatalf("unexpected artifacts from lineage: %#v", artifacts)
}
}
func TestBuildNodeEnvDoesNotAllowHyperparamsToOverrideRunTimestampEnv(t *testing.T) {
run := &domain.RunRecord{
ID: "run-2",
WorkflowID: "workflow-2",
Trigger: domain.RunTriggerScheduled,
PreviousRunAt: "2026-04-21T10:00:00Z",
CurrentRunAt: "2026-04-21T11:30:00Z",
}
workflow := &domain.WorkflowRecord{
ID: "workflow-2",
Hyperparams: []domain.HyperParam{
{Key: "WORKFLOW_PREVIOUS_RUN_TIMESTAMP", Value: "workflow-999"},
{Key: "WORKFLOW_CURRENT_RUN_TIMESTAMP", Value: "workflow-888"},
},
}
node := domain.WorkflowNode{
ID: "node-2",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Description: "exec",
Hyperparams: []domain.HyperParam{
{Key: "WORKFLOW_PREVIOUS_RUN_TIMESTAMP", Value: "999"},
{Key: "WORKFLOW_CURRENT_RUN_TIMESTAMP", Value: "888"},
},
},
}
workspace := nodeWorkspace{
RunRoot: "/tmp/workspace/system/runs/run-2",
Root: "/tmp/workspace/system/runs/run-2/nodes/node-2",
OutputDir: "/tmp/workspace/system/runs/run-2/nodes/node-2/outputs",
InputArtifactsPath: "/tmp/workspace/system/runs/run-2/nodes/node-2/input-artifacts.json",
ResultPath: "/tmp/workspace/system/runs/run-2/nodes/node-2/result.json",
StageManifestPath: "/tmp/workspace/system/runs/run-2/nodes/node-2/stage-manifest.json",
}
env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != "2026-04-21T18:00:00+08:00" {
t.Fatalf("expected protected previous timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"])
}
if env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] != "2026-04-21T19:30:00+08:00" {
t.Fatalf("expected protected current timestamp env, got %q", env["WORKFLOW_CURRENT_RUN_TIMESTAMP"])
}
}
func TestBuildNodeEnvInjectsProtectedOperatorCommandPolicy(t *testing.T) {
t.Setenv("OPERATOR_ALLOW_SHELL_COMMANDS", "false")
t.Setenv("OPERATOR_COMMAND_ALLOWLIST_REGEX", `^python3?\s`)
t.Setenv("OPERATOR_COMMAND_DENYLIST_REGEX", `(^|\s)rm\s+-rf`)
run := &domain.RunRecord{
ID: "run-3",
WorkflowID: "workflow-3",
Trigger: domain.RunTriggerManual,
}
workflow := &domain.WorkflowRecord{
ID: "workflow-3",
Hyperparams: []domain.HyperParam{
{Key: "OPERATOR_ALLOW_SHELL_COMMANDS", Value: "true"},
{Key: "OPERATOR_COMMAND_ALLOWLIST_REGEX", Value: ".*"},
},
}
node := domain.WorkflowNode{
ID: "node-3",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Shell 命令执行",
Category: "FLOW",
Hyperparams: []domain.HyperParam{
{Key: "OPERATOR_COMMAND_DENYLIST_REGEX", Value: ""},
},
},
}
workspace := nodeWorkspace{
RunRoot: "/tmp/workspace/system/runs/run-3",
Root: "/tmp/workspace/system/runs/run-3/nodes/node-3",
OutputDir: "/tmp/workspace/system/runs/run-3/nodes/node-3/outputs",
InputArtifactsPath: "/tmp/workspace/system/runs/run-3/nodes/node-3/input-artifacts.json",
ResultPath: "/tmp/workspace/system/runs/run-3/nodes/node-3/result.json",
StageManifestPath: "/tmp/workspace/system/runs/run-3/nodes/node-3/stage-manifest.json",
}
env, err := buildNodeEnv(run, workflow, node, nil, workspace, &domain.SettingsRecord{})
if err != nil {
t.Fatalf("buildNodeEnv returned error: %v", err)
}
if env["OPERATOR_ALLOW_SHELL_COMMANDS"] != "false" {
t.Fatalf("expected platform shell command flag to win, got %q", env["OPERATOR_ALLOW_SHELL_COMMANDS"])
}
if env["OPERATOR_COMMAND_ALLOWLIST_REGEX"] != `^python3?\s` {
t.Fatalf("expected platform allowlist to win, got %q", env["OPERATOR_COMMAND_ALLOWLIST_REGEX"])
}
if env["OPERATOR_COMMAND_DENYLIST_REGEX"] != `(^|\s)rm\s+-rf` {
t.Fatalf("expected platform denylist to win, got %q", env["OPERATOR_COMMAND_DENYLIST_REGEX"])
}
}
func TestParseExecutionOutputReadsResultFilePayload(t *testing.T) {
resultPath := filepath.Join(t.TempDir(), "result.json")
payload := executionResult{
Logs: []domain.LogEntryRecord{{Level: "INFO", Message: "done", Operator: "测试节点"}},
Artifacts: []domain.ExecutionArtifact{{ID: "artifact-1", Name: "report.json"}},
}
data, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
if err := os.WriteFile(resultPath, data, 0o644); err != nil {
t.Fatalf("write payload: %v", err)
}
parsed, err := parseExecutionOutput("some log\n"+resultPathPrefix+resultPath+"\n", "测试节点")
if err != nil {
t.Fatalf("parseExecutionOutput returned error: %v", err)
}
if len(parsed.Artifacts) != 1 || parsed.Artifacts[0].ID != "artifact-1" {
t.Fatalf("unexpected parsed artifacts: %#v", parsed.Artifacts)
}
}
func TestLoadStageManifestDataRefs(t *testing.T) {
stageManifestPath := filepath.Join(t.TempDir(), "stage-manifest.json")
payload := map[string]any{
"dataRefs": []map[string]any{
{
"artifactId": "artifact-1",
"artifactName": "dataset-index.json",
"kind": "dataset",
"datasetType": "x-platform-index",
"manifestPath": "/workspace/run/stage/manifest.json",
"payloadPath": "/workspace/run/stage/payload.json",
"partitionDir": "/workspace/run/stage/partitions",
"partitionCount": 3,
"itemCount": 12,
},
},
}
data, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
if err := os.WriteFile(stageManifestPath, data, 0o644); err != nil {
t.Fatalf("write stage manifest: %v", err)
}
dataRefs, err := loadStageManifestDataRefs(stageManifestPath)
if err != nil {
t.Fatalf("loadStageManifestDataRefs returned error: %v", err)
}
if len(dataRefs) != 1 || dataRefs[0].DatasetType != "x-platform-index" {
t.Fatalf("unexpected data refs: %#v", dataRefs)
}
}