droboticflow/tests/backend/poc_gpu_workflow_acceptance_script_test.py
2026-07-07 18:57:49 +08:00

333 lines
12 KiB
Python

import json
import os
import pathlib
import subprocess
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "poc-gpu-workflow-blocked-acceptance.sh"
def test_poc_gpu_workflow_acceptance_uses_public_load_balancer_and_expected_failure(tmp_path):
captured_workflow = tmp_path / "workflow.json"
stub_bin = tmp_path / "bin"
stub_bin.mkdir()
kubectl = stub_bin / "kubectl"
kubectl.write_text(
"""#!/usr/bin/env python3
import json
import sys
args = sys.argv[1:]
if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]:
print(json.dumps({
"status": {
"loadBalancer": {
"ingress": [
{"ip": "192.168.0.74"},
{"ip": "120.48.164.19"}
]
}
}
}))
raise SystemExit(0)
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
raise SystemExit(97)
""",
encoding="utf-8",
)
kubectl.chmod(0o755)
curl = stub_bin / "curl"
curl.write_text(
f"""#!/usr/bin/env python3
import json
import pathlib
import sys
args = sys.argv[1:]
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
if url and not url.startswith("http://120.48.164.19"):
raise SystemExit(f"expected public load balancer URL, got {{url}}")
body = ""
for index, arg in enumerate(args):
if arg == "-d" and index + 1 < len(args):
body = args[index + 1]
if url.endswith("/api/health"):
payload = {{"status": "ok"}}
elif url.endswith("/api/auth/login"):
payload = {{"email": "operator@example.test"}}
elif url.endswith("/api/projects"):
payload = {{"id": "project-1"}}
elif url.endswith("/api/projects/project-1/workflows"):
pathlib.Path({str(captured_workflow)!r}).write_text(body, encoding="utf-8")
payload = {{"id": "workflow-1"}}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {{"id": "run-1"}}
elif url.endswith("/api/runs/run-1/logs"):
payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {{"items": []}}
elif url.endswith("/api/runs/run-1"):
payload = {{"status": "failed"}}
else:
raise SystemExit(f"unexpected URL: {{url}}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
curl.chmod(0o755)
env = os.environ.copy()
env.update(
{
"DEPLOY_ENV": "poc",
"DROBOTICFLOW_ACCEPTANCE_EMAIL": "operator@example.test",
"DROBOTICFLOW_ACCEPTANCE_PASSWORD": "secret",
"K8S_NAMESPACE": "droboticflow",
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
"POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "false",
"POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
}
)
env.pop("API_BASE_URL", None)
env.pop("BASE_URL", None)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
assert result.returncode == 0, result.stderr + result.stdout
assert "run status: failed" in result.stdout
workflow = json.loads(captured_workflow.read_text(encoding="utf-8"))
source_node = next(node for node in workflow["nodes"] if node["id"] == "source-1")
assert source_node["data"]["gpu"] == "1"
def test_poc_gpu_workflow_acceptance_uses_token_endpoint_and_worker_port_forward(tmp_path):
captured_acceptance = tmp_path / "acceptance.json"
run_status = tmp_path / "run-status.txt"
stub_bin = tmp_path / "bin"
stub_bin.mkdir()
kubectl = stub_bin / "kubectl"
kubectl.write_text(
"""#!/usr/bin/env python3
import json
import sys
import time
args = sys.argv[1:]
if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]:
print(json.dumps({
"status": {
"loadBalancer": {
"ingress": [
{"ip": "192.168.0.74"},
{"ip": "120.48.164.19"}
]
}
}
}))
raise SystemExit(0)
if args == ["-n", "droboticflow", "get", "secret", "droboticflow-auth-secret", "-o", "json"]:
print(json.dumps({
"data": {
"DROBOTICFLOW_API_TOKEN": "c2hhcmVkLXRva2Vu"
}
}))
raise SystemExit(0)
if len(args) >= 5 and args[:4] == ["-n", "droboticflow", "port-forward", "service/droboticflow-worker"]:
print("Forwarding from 127.0.0.1 -> 3002", flush=True)
time.sleep(30)
raise SystemExit(0)
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
raise SystemExit(97)
""",
encoding="utf-8",
)
kubectl.chmod(0o755)
curl = stub_bin / "curl"
curl.write_text(
f"""#!/usr/bin/env python3
import json
import pathlib
import sys
status_file = pathlib.Path({str(run_status)!r})
captured_acceptance = pathlib.Path({str(captured_acceptance)!r})
args = sys.argv[1:]
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
headers = [args[index + 1] for index, arg in enumerate(args[:-1]) if arg == "-H"]
body = ""
for index, arg in enumerate(args):
if arg == "-d" and index + 1 < len(args):
body = args[index + 1]
if url.startswith("http://120.48.164.19") and "Authorization: Bearer shared-token" not in headers:
raise SystemExit("expected bearer token for API call")
if url.startswith("http://") and "120.48.164.19" not in url and "127.0.0.1" not in url:
raise SystemExit(f"expected public load balancer or worker port-forward URL, got {{url}}")
if url.endswith("/api/health"):
payload = {{"status": "ok"}}
elif url.endswith("/api/acceptance/poc-gpu-workflow"):
captured_acceptance.write_text(body, encoding="utf-8")
status_file.write_text("queued", encoding="utf-8")
payload = {{
"project": {{"id": "project-token", "name": "POC GPU Workflow Acceptance"}},
"workflow": {{"id": "workflow-token", "name": "POC GPU Unschedulable Acceptance Workflow"}},
"run": {{"id": "run-token", "status": "queued"}}
}}
elif url.endswith("/worker-api/health"):
payload = {{"status": "ok", "service": "droboticflow-worker"}}
elif url.endswith("/worker-api/process-once"):
status_file.write_text("failed", encoding="utf-8")
payload = {{"run": {{"id": "run-token", "status": "failed"}}}}
elif url.endswith("/api/runs/run-token/logs"):
payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}}
elif url.endswith("/api/runs/run-token"):
payload = {{
"id": "run-token",
"status": status_file.read_text(encoding="utf-8").strip(),
"resourceSeries": [{{"timestamp": "2026-06-28T00:00:00Z", "cpu": 1, "memory": 64, "gpu": 3, "gpuMemoryUsedMiB": 1024, "gpuMemoryTotalMiB": 81920}}],
}}
else:
raise SystemExit(f"unexpected URL: {{url}}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
curl.chmod(0o755)
env = os.environ.copy()
env.update(
{
"DEPLOY_ENV": "poc",
"K8S_NAMESPACE": "droboticflow",
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
"POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "false",
"POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
}
)
env.pop("API_BASE_URL", None)
env.pop("BASE_URL", None)
env.pop("DROBOTICFLOW_API_TOKEN", None)
env.pop("DROBOTICFLOW_ACCEPTANCE_EMAIL", None)
env.pop("DROBOTICFLOW_ACCEPTANCE_PASSWORD", None)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
assert result.returncode == 0, result.stderr + result.stdout
assert "created POC GPU acceptance project=project-token workflow=workflow-token run=run-token" in result.stdout
assert "worker port-forward:" in result.stdout
assert "run status: failed" in result.stdout
body = json.loads(captured_acceptance.read_text(encoding="utf-8"))
assert body == {"sourcePath": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform"}
def test_poc_gpu_workflow_acceptance_can_expect_completed_gpu_run(tmp_path):
captured_acceptance = tmp_path / "acceptance.json"
run_status = tmp_path / "run-status.txt"
stub_bin = tmp_path / "bin"
stub_bin.mkdir()
kubectl = stub_bin / "kubectl"
kubectl.write_text(
"""#!/usr/bin/env python3
import json
import sys
import time
args = sys.argv[1:]
if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]:
print(json.dumps({"status": {"loadBalancer": {"ingress": [{"ip": "120.48.164.19"}]}}}))
raise SystemExit(0)
if args == ["-n", "droboticflow", "get", "secret", "droboticflow-auth-secret", "-o", "json"]:
print(json.dumps({"data": {"DROBOTICFLOW_API_TOKEN": "c2hhcmVkLXRva2Vu"}}))
raise SystemExit(0)
if len(args) >= 5 and args[:4] == ["-n", "droboticflow", "port-forward", "service/droboticflow-worker"]:
print("Forwarding from 127.0.0.1 -> 3002", flush=True)
time.sleep(30)
raise SystemExit(0)
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
raise SystemExit(97)
""",
encoding="utf-8",
)
kubectl.chmod(0o755)
curl = stub_bin / "curl"
curl.write_text(
f"""#!/usr/bin/env python3
import json
import pathlib
import sys
status_file = pathlib.Path({str(run_status)!r})
captured_acceptance = pathlib.Path({str(captured_acceptance)!r})
args = sys.argv[1:]
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
body = ""
for index, arg in enumerate(args):
if arg == "-d" and index + 1 < len(args):
body = args[index + 1]
if url.endswith("/api/health"):
payload = {{"status": "ok"}}
elif url.endswith("/api/acceptance/poc-gpu-workflow"):
captured_acceptance.write_text(body, encoding="utf-8")
status_file.write_text("queued", encoding="utf-8")
payload = {{"project": {{"id": "project-token"}}, "workflow": {{"id": "workflow-token"}}, "run": {{"id": "run-token", "status": "queued"}}}}
elif url.endswith("/worker-api/health"):
payload = {{"status": "ok"}}
elif url.endswith("/worker-api/process-once"):
status_file.write_text("completed", encoding="utf-8")
payload = {{"run": {{"id": "run-token", "status": "completed"}}}}
elif url.endswith("/api/runs/run-token/logs"):
payload = {{"items": [{{"message": "GPU operator completed"}}]}}
elif url.endswith("/api/runs/run-token/artifacts"):
payload = {{"items": [{{"name": "manifest.json", "type": "json", "size": 128}}]}}
elif url.endswith("/api/runs/run-token"):
payload = {{
"id": "run-token",
"status": status_file.read_text(encoding="utf-8").strip(),
"resourceSeries": [{{"timestamp": "2026-06-28T00:00:00Z", "cpu": 1, "memory": 64, "gpu": 3, "gpuMemoryUsedMiB": 1024, "gpuMemoryTotalMiB": 81920}}],
}}
else:
raise SystemExit(f"unexpected URL: {{url}}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
curl.chmod(0o755)
env = os.environ.copy()
env.update(
{
"DEPLOY_ENV": "poc",
"K8S_NAMESPACE": "droboticflow",
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
"POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "true",
"POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
}
)
env.pop("API_BASE_URL", None)
env.pop("BASE_URL", None)
env.pop("DROBOTICFLOW_API_TOKEN", None)
env.pop("DROBOTICFLOW_ACCEPTANCE_EMAIL", None)
env.pop("DROBOTICFLOW_ACCEPTANCE_PASSWORD", None)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
assert result.returncode == 0, result.stderr + result.stdout
assert "run status: completed" in result.stdout
assert "artifacts summary:" in result.stdout
body = json.loads(captured_acceptance.read_text(encoding="utf-8"))
assert body == {"sourcePath": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform"}