droboticflow/tests/backend/live_workflow_acceptance_script_test.py
2026-06-28 19:47:06 +08:00

283 lines
9.6 KiB
Python

import json
import os
import pathlib
import subprocess
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "live-workflow-acceptance.sh"
def run_acceptance(extra_env: dict[str, str]) -> subprocess.CompletedProcess:
env = os.environ.copy()
for key in (
"API_BASE_URL",
"BASE_URL",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH",
"WORKER_API_BASE_URL",
"WORKER_BASE_URL",
"DROBOTICFLOW_API_TOKEN",
):
env.pop(key, None)
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def test_acceptance_requires_explicit_endpoint_when_deployed_mode_is_enabled():
result = run_acceptance(
{
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 1
assert "API_BASE_URL or BASE_URL is required when WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true" in result.stderr
def test_acceptance_rejects_invalid_explicit_endpoint_mode_boolean():
result = run_acceptance(
{
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "maybe",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 1
assert "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean" in result.stderr
def test_acceptance_allows_public_api_without_token(tmp_path):
fake_curl = tmp_path / "curl"
fake_curl.write_text(
"""#!/usr/bin/env python3
import json
import sys
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
if url.endswith("/api/health"):
payload = {"status": "ok"}
elif url.endswith("/api/projects"):
payload = {"id": "project-1"}
elif url.endswith("/api/projects/project-1/workflows"):
payload = {"id": "workflow-1"}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {"id": "run-1"}
elif url.endswith("/api/runs/run-1/logs"):
payload = {"items": [{"message": "ok"}]}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {"items": [{"name": "source-manifest.json"}]}
elif url.endswith("/api/runs/run-1"):
payload = {
"status": "completed",
"nodeExecutions": [
{
"nodeId": "source-1",
"status": "completed",
"resourceSeries": [],
},
{
"nodeId": "upload-1",
"status": "completed",
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
"inputLineage": [
{
"sourceNodeId": "source-1",
"sourcePortId": "out-1",
"targetNodeId": "upload-1",
"targetPortId": "in-1",
"artifactId": "artifact-1",
}
],
"resourceSeries": [],
},
],
}
else:
raise SystemExit(f"unexpected URL: {url}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
fake_curl.chmod(0o755)
result = run_acceptance(
{
"API_BASE_URL": "http://poc.example.test",
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 0, result.stderr
assert "run status: completed" in result.stdout
def test_acceptance_requires_completed_workflow_lineage(tmp_path):
fake_curl = tmp_path / "curl"
fake_curl.write_text(
"""#!/usr/bin/env python3
import json
import sys
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
if url.endswith("/api/health"):
payload = {"status": "ok"}
elif url.endswith("/api/projects"):
payload = {"id": "project-1"}
elif url.endswith("/api/projects/project-1/workflows"):
payload = {"id": "workflow-1"}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {"id": "run-1"}
elif url.endswith("/api/runs/run-1/logs"):
payload = {"items": [{"message": "ok"}]}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {"items": [{"name": "source-manifest.json"}]}
elif url.endswith("/api/runs/run-1"):
payload = {"status": "completed", "nodeExecutions": [{"nodeId": "upload-1", "status": "completed"}]}
else:
raise SystemExit(f"unexpected URL: {url}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
fake_curl.chmod(0o755)
result = run_acceptance(
{
"API_BASE_URL": "http://poc.example.test",
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 1
assert "expected source-1 to upload-1 inputLineage artifact" in result.stderr
def test_acceptance_requires_gpu_memory_when_completed_gpu_workflow(tmp_path):
fake_curl = tmp_path / "curl"
fake_curl.write_text(
"""#!/usr/bin/env python3
import json
import sys
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
if url.endswith("/api/health"):
payload = {"status": "ok"}
elif url.endswith("/api/projects"):
payload = {"id": "project-1"}
elif url.endswith("/api/projects/project-1/workflows"):
payload = {"id": "workflow-1"}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {"id": "run-1"}
elif url.endswith("/api/runs/run-1/logs"):
payload = {"items": [{"message": "ok"}]}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {"items": [{"name": "source-manifest.json"}]}
elif url.endswith("/api/runs/run-1"):
payload = {
"status": "completed",
"nodeExecutions": [
{
"nodeId": "upload-1",
"status": "completed",
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
"inputLineage": [
{
"sourceNodeId": "source-1",
"targetNodeId": "upload-1",
"artifactId": "artifact-1",
}
],
}
],
}
else:
raise SystemExit(f"unexpected URL: {url}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
fake_curl.chmod(0o755)
result = run_acceptance(
{
"API_BASE_URL": "http://poc.example.test",
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "completed",
"WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 1
assert "expected GPU memory metrics in completed GPU workflow resourceSeries" in result.stderr
def test_acceptance_can_expect_gpu_preflight_failure(tmp_path):
captured_workflow = tmp_path / "workflow.json"
fake_curl = tmp_path / "curl"
fake_curl.write_text(
f"""#!/usr/bin/env python3
import json
import pathlib
import sys
args = sys.argv[1:]
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
body = ""
for index, arg in enumerate(args):
if arg == "-d" and index + 1 < len(args):
body = args[index + 1]
if url.endswith("/api/health"):
payload = {{"status": "ok"}}
elif url.endswith("/api/projects"):
payload = {{"id": "project-1"}}
elif url.endswith("/api/projects/project-1/workflows"):
pathlib.Path({str(captured_workflow)!r}).write_text(body, encoding="utf-8")
payload = {{"id": "workflow-1"}}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {{"id": "run-1"}}
elif url.endswith("/api/runs/run-1/logs"):
payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {{"items": []}}
elif url.endswith("/api/runs/run-1"):
payload = {{"status": "failed"}}
else:
raise SystemExit(f"unexpected URL: {{url}}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
fake_curl.chmod(0o755)
result = run_acceptance(
{
"API_BASE_URL": "http://poc.example.test",
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING": "gpu request cannot be scheduled",
"WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "failed",
"WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 0, result.stderr
assert "run status: failed" in result.stdout
workflow = json.loads(captured_workflow.read_text(encoding="utf-8"))
source_node = next(node for node in workflow["nodes"] if node["id"] == "source-1")
assert source_node["data"]["gpu"] == "1"