import json import os import pathlib import subprocess ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "live-workflow-acceptance.sh" def run_acceptance(extra_env: dict[str, str]) -> subprocess.CompletedProcess: env = os.environ.copy() for key in ( "API_BASE_URL", "BASE_URL", "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS", "WORKFLOW_ACCEPTANCE_SOURCE_PATH", "WORKER_API_BASE_URL", "WORKER_BASE_URL", "DROBOTICFLOW_API_TOKEN", ): env.pop(key, None) env.update(extra_env) return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) def test_acceptance_requires_explicit_endpoint_when_deployed_mode_is_enabled(): result = run_acceptance( { "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", } ) assert result.returncode == 1 assert "API_BASE_URL or BASE_URL is required when WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true" in result.stderr def test_acceptance_rejects_invalid_explicit_endpoint_mode_boolean(): result = run_acceptance( { "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "maybe", "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", } ) assert result.returncode == 1 assert "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean" in result.stderr def test_acceptance_allows_public_api_without_token(tmp_path): fake_curl = tmp_path / "curl" fake_curl.write_text( """#!/usr/bin/env python3 import json import sys url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "") if url.endswith("/api/health"): payload = {"status": "ok"} elif url.endswith("/api/projects"): payload = {"id": "project-1"} elif url.endswith("/api/projects/project-1/workflows"): payload = {"id": "workflow-1"} elif url.endswith("/api/workflows/workflow-1/run"): payload = {"id": "run-1"} elif url.endswith("/api/runs/run-1/logs"): payload = {"items": [{"message": "ok"}]} elif url.endswith("/api/runs/run-1/artifacts"): payload = {"items": [{"name": "source-manifest.json"}]} elif url.endswith("/api/runs/run-1"): payload = { "status": "completed", "nodeExecutions": [ { "nodeId": "source-1", "status": "completed", "resourceSeries": [], }, { "nodeId": "upload-1", "status": "completed", "inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}], "inputLineage": [ { "sourceNodeId": "source-1", "sourcePortId": "out-1", "targetNodeId": "upload-1", "targetPortId": "in-1", "artifactId": "artifact-1", } ], "resourceSeries": [], }, ], } else: raise SystemExit(f"unexpected URL: {url}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) fake_curl.chmod(0o755) result = run_acceptance( { "API_BASE_URL": "http://poc.example.test", "PATH": f"{tmp_path}:{os.environ['PATH']}", "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", } ) assert result.returncode == 0, result.stderr assert "run status: completed" in result.stdout def test_acceptance_requires_completed_workflow_lineage(tmp_path): fake_curl = tmp_path / "curl" fake_curl.write_text( """#!/usr/bin/env python3 import json import sys url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "") if url.endswith("/api/health"): payload = {"status": "ok"} elif url.endswith("/api/projects"): payload = {"id": "project-1"} elif url.endswith("/api/projects/project-1/workflows"): payload = {"id": "workflow-1"} elif url.endswith("/api/workflows/workflow-1/run"): payload = {"id": "run-1"} elif url.endswith("/api/runs/run-1/logs"): payload = {"items": [{"message": "ok"}]} elif url.endswith("/api/runs/run-1/artifacts"): payload = {"items": [{"name": "source-manifest.json"}]} elif url.endswith("/api/runs/run-1"): payload = {"status": "completed", "nodeExecutions": [{"nodeId": "upload-1", "status": "completed"}]} else: raise SystemExit(f"unexpected URL: {url}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) fake_curl.chmod(0o755) result = run_acceptance( { "API_BASE_URL": "http://poc.example.test", "PATH": f"{tmp_path}:{os.environ['PATH']}", "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", } ) assert result.returncode == 1 assert "expected source-1 to upload-1 inputLineage artifact" in result.stderr def test_acceptance_requires_gpu_memory_when_completed_gpu_workflow(tmp_path): fake_curl = tmp_path / "curl" fake_curl.write_text( """#!/usr/bin/env python3 import json import sys url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "") if url.endswith("/api/health"): payload = {"status": "ok"} elif url.endswith("/api/projects"): payload = {"id": "project-1"} elif url.endswith("/api/projects/project-1/workflows"): payload = {"id": "workflow-1"} elif url.endswith("/api/workflows/workflow-1/run"): payload = {"id": "run-1"} elif url.endswith("/api/runs/run-1/logs"): payload = {"items": [{"message": "ok"}]} elif url.endswith("/api/runs/run-1/artifacts"): payload = {"items": [{"name": "source-manifest.json"}]} elif url.endswith("/api/runs/run-1"): payload = { "status": "completed", "nodeExecutions": [ { "nodeId": "upload-1", "status": "completed", "inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}], "inputLineage": [ { "sourceNodeId": "source-1", "targetNodeId": "upload-1", "artifactId": "artifact-1", } ], } ], } else: raise SystemExit(f"unexpected URL: {url}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) fake_curl.chmod(0o755) result = run_acceptance( { "API_BASE_URL": "http://poc.example.test", "PATH": f"{tmp_path}:{os.environ['PATH']}", "WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "completed", "WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1", "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", } ) assert result.returncode == 1 assert "expected GPU memory metrics in completed GPU workflow resourceSeries" in result.stderr def test_acceptance_can_expect_gpu_preflight_failure(tmp_path): captured_workflow = tmp_path / "workflow.json" fake_curl = tmp_path / "curl" fake_curl.write_text( f"""#!/usr/bin/env python3 import json import pathlib import sys args = sys.argv[1:] url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "") body = "" for index, arg in enumerate(args): if arg == "-d" and index + 1 < len(args): body = args[index + 1] if url.endswith("/api/health"): payload = {{"status": "ok"}} elif url.endswith("/api/projects"): payload = {{"id": "project-1"}} elif url.endswith("/api/projects/project-1/workflows"): pathlib.Path({str(captured_workflow)!r}).write_text(body, encoding="utf-8") payload = {{"id": "workflow-1"}} elif url.endswith("/api/workflows/workflow-1/run"): payload = {{"id": "run-1"}} elif url.endswith("/api/runs/run-1/logs"): payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}} elif url.endswith("/api/runs/run-1/artifacts"): payload = {{"items": []}} elif url.endswith("/api/runs/run-1"): payload = {{"status": "failed"}} else: raise SystemExit(f"unexpected URL: {{url}}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) fake_curl.chmod(0o755) result = run_acceptance( { "API_BASE_URL": "http://poc.example.test", "PATH": f"{tmp_path}:{os.environ['PATH']}", "WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING": "gpu request cannot be scheduled", "WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "failed", "WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1", "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", } ) assert result.returncode == 0, result.stderr assert "run status: failed" in result.stdout workflow = json.loads(captured_workflow.read_text(encoding="utf-8")) source_node = next(node for node in workflow["nodes"] if node["id"] == "source-1") assert source_node["data"]["gpu"] == "1"