283 lines
9.6 KiB
Python
283 lines
9.6 KiB
Python
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "live-workflow-acceptance.sh"
|
|
|
|
|
|
def run_acceptance(extra_env: dict[str, str]) -> subprocess.CompletedProcess:
|
|
env = os.environ.copy()
|
|
for key in (
|
|
"API_BASE_URL",
|
|
"BASE_URL",
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH",
|
|
"WORKER_API_BASE_URL",
|
|
"WORKER_BASE_URL",
|
|
"DROBOTICFLOW_API_TOKEN",
|
|
):
|
|
env.pop(key, None)
|
|
env.update(extra_env)
|
|
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
|
|
|
|
def test_acceptance_requires_explicit_endpoint_when_deployed_mode_is_enabled():
|
|
result = run_acceptance(
|
|
{
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
|
}
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "API_BASE_URL or BASE_URL is required when WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT=true" in result.stderr
|
|
|
|
|
|
def test_acceptance_rejects_invalid_explicit_endpoint_mode_boolean():
|
|
result = run_acceptance(
|
|
{
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "maybe",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
|
}
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean" in result.stderr
|
|
|
|
|
|
def test_acceptance_allows_public_api_without_token(tmp_path):
|
|
fake_curl = tmp_path / "curl"
|
|
fake_curl.write_text(
|
|
"""#!/usr/bin/env python3
|
|
import json
|
|
import sys
|
|
|
|
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
if url.endswith("/api/health"):
|
|
payload = {"status": "ok"}
|
|
elif url.endswith("/api/projects"):
|
|
payload = {"id": "project-1"}
|
|
elif url.endswith("/api/projects/project-1/workflows"):
|
|
payload = {"id": "workflow-1"}
|
|
elif url.endswith("/api/workflows/workflow-1/run"):
|
|
payload = {"id": "run-1"}
|
|
elif url.endswith("/api/runs/run-1/logs"):
|
|
payload = {"items": [{"message": "ok"}]}
|
|
elif url.endswith("/api/runs/run-1/artifacts"):
|
|
payload = {"items": [{"name": "source-manifest.json"}]}
|
|
elif url.endswith("/api/runs/run-1"):
|
|
payload = {
|
|
"status": "completed",
|
|
"nodeExecutions": [
|
|
{
|
|
"nodeId": "source-1",
|
|
"status": "completed",
|
|
"resourceSeries": [],
|
|
},
|
|
{
|
|
"nodeId": "upload-1",
|
|
"status": "completed",
|
|
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
|
|
"inputLineage": [
|
|
{
|
|
"sourceNodeId": "source-1",
|
|
"sourcePortId": "out-1",
|
|
"targetNodeId": "upload-1",
|
|
"targetPortId": "in-1",
|
|
"artifactId": "artifact-1",
|
|
}
|
|
],
|
|
"resourceSeries": [],
|
|
},
|
|
],
|
|
}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {url}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_curl.chmod(0o755)
|
|
|
|
result = run_acceptance(
|
|
{
|
|
"API_BASE_URL": "http://poc.example.test",
|
|
"PATH": f"{tmp_path}:{os.environ['PATH']}",
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
|
}
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "run status: completed" in result.stdout
|
|
|
|
|
|
def test_acceptance_requires_completed_workflow_lineage(tmp_path):
|
|
fake_curl = tmp_path / "curl"
|
|
fake_curl.write_text(
|
|
"""#!/usr/bin/env python3
|
|
import json
|
|
import sys
|
|
|
|
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
if url.endswith("/api/health"):
|
|
payload = {"status": "ok"}
|
|
elif url.endswith("/api/projects"):
|
|
payload = {"id": "project-1"}
|
|
elif url.endswith("/api/projects/project-1/workflows"):
|
|
payload = {"id": "workflow-1"}
|
|
elif url.endswith("/api/workflows/workflow-1/run"):
|
|
payload = {"id": "run-1"}
|
|
elif url.endswith("/api/runs/run-1/logs"):
|
|
payload = {"items": [{"message": "ok"}]}
|
|
elif url.endswith("/api/runs/run-1/artifacts"):
|
|
payload = {"items": [{"name": "source-manifest.json"}]}
|
|
elif url.endswith("/api/runs/run-1"):
|
|
payload = {"status": "completed", "nodeExecutions": [{"nodeId": "upload-1", "status": "completed"}]}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {url}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_curl.chmod(0o755)
|
|
|
|
result = run_acceptance(
|
|
{
|
|
"API_BASE_URL": "http://poc.example.test",
|
|
"PATH": f"{tmp_path}:{os.environ['PATH']}",
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
|
}
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "expected source-1 to upload-1 inputLineage artifact" in result.stderr
|
|
|
|
|
|
def test_acceptance_requires_gpu_memory_when_completed_gpu_workflow(tmp_path):
|
|
fake_curl = tmp_path / "curl"
|
|
fake_curl.write_text(
|
|
"""#!/usr/bin/env python3
|
|
import json
|
|
import sys
|
|
|
|
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
if url.endswith("/api/health"):
|
|
payload = {"status": "ok"}
|
|
elif url.endswith("/api/projects"):
|
|
payload = {"id": "project-1"}
|
|
elif url.endswith("/api/projects/project-1/workflows"):
|
|
payload = {"id": "workflow-1"}
|
|
elif url.endswith("/api/workflows/workflow-1/run"):
|
|
payload = {"id": "run-1"}
|
|
elif url.endswith("/api/runs/run-1/logs"):
|
|
payload = {"items": [{"message": "ok"}]}
|
|
elif url.endswith("/api/runs/run-1/artifacts"):
|
|
payload = {"items": [{"name": "source-manifest.json"}]}
|
|
elif url.endswith("/api/runs/run-1"):
|
|
payload = {
|
|
"status": "completed",
|
|
"nodeExecutions": [
|
|
{
|
|
"nodeId": "upload-1",
|
|
"status": "completed",
|
|
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
|
|
"inputLineage": [
|
|
{
|
|
"sourceNodeId": "source-1",
|
|
"targetNodeId": "upload-1",
|
|
"artifactId": "artifact-1",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {url}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_curl.chmod(0o755)
|
|
|
|
result = run_acceptance(
|
|
{
|
|
"API_BASE_URL": "http://poc.example.test",
|
|
"PATH": f"{tmp_path}:{os.environ['PATH']}",
|
|
"WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "completed",
|
|
"WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1",
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
|
}
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "expected GPU memory metrics in completed GPU workflow resourceSeries" in result.stderr
|
|
|
|
|
|
def test_acceptance_can_expect_gpu_preflight_failure(tmp_path):
|
|
captured_workflow = tmp_path / "workflow.json"
|
|
fake_curl = tmp_path / "curl"
|
|
fake_curl.write_text(
|
|
f"""#!/usr/bin/env python3
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
args = sys.argv[1:]
|
|
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
body = ""
|
|
for index, arg in enumerate(args):
|
|
if arg == "-d" and index + 1 < len(args):
|
|
body = args[index + 1]
|
|
|
|
if url.endswith("/api/health"):
|
|
payload = {{"status": "ok"}}
|
|
elif url.endswith("/api/projects"):
|
|
payload = {{"id": "project-1"}}
|
|
elif url.endswith("/api/projects/project-1/workflows"):
|
|
pathlib.Path({str(captured_workflow)!r}).write_text(body, encoding="utf-8")
|
|
payload = {{"id": "workflow-1"}}
|
|
elif url.endswith("/api/workflows/workflow-1/run"):
|
|
payload = {{"id": "run-1"}}
|
|
elif url.endswith("/api/runs/run-1/logs"):
|
|
payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}}
|
|
elif url.endswith("/api/runs/run-1/artifacts"):
|
|
payload = {{"items": []}}
|
|
elif url.endswith("/api/runs/run-1"):
|
|
payload = {{"status": "failed"}}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {{url}}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_curl.chmod(0o755)
|
|
|
|
result = run_acceptance(
|
|
{
|
|
"API_BASE_URL": "http://poc.example.test",
|
|
"PATH": f"{tmp_path}:{os.environ['PATH']}",
|
|
"WORKFLOW_ACCEPTANCE_EXPECT_LOG_SUBSTRING": "gpu request cannot be scheduled",
|
|
"WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "failed",
|
|
"WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1",
|
|
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
|
}
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "run status: failed" in result.stdout
|
|
workflow = json.loads(captured_workflow.read_text(encoding="utf-8"))
|
|
source_node = next(node for node in workflow["nodes"] if node["id"] == "source-1")
|
|
assert source_node["data"]["gpu"] == "1"
|