import json import os import pathlib import subprocess ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "poc-gpu-workflow-blocked-acceptance.sh" def test_poc_gpu_workflow_acceptance_uses_public_load_balancer_and_expected_failure(tmp_path): captured_workflow = tmp_path / "workflow.json" stub_bin = tmp_path / "bin" stub_bin.mkdir() kubectl = stub_bin / "kubectl" kubectl.write_text( """#!/usr/bin/env python3 import json import sys args = sys.argv[1:] if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]: print(json.dumps({ "status": { "loadBalancer": { "ingress": [ {"ip": "192.168.0.74"}, {"ip": "120.48.164.19"} ] } } })) raise SystemExit(0) print("unexpected kubectl call: " + " ".join(args), file=sys.stderr) raise SystemExit(97) """, encoding="utf-8", ) kubectl.chmod(0o755) curl = stub_bin / "curl" curl.write_text( f"""#!/usr/bin/env python3 import json import pathlib import sys args = sys.argv[1:] url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "") if url and not url.startswith("http://120.48.164.19"): raise SystemExit(f"expected public load balancer URL, got {{url}}") body = "" for index, arg in enumerate(args): if arg == "-d" and index + 1 < len(args): body = args[index + 1] if url.endswith("/api/health"): payload = {{"status": "ok"}} elif url.endswith("/api/auth/login"): payload = {{"email": "operator@example.test"}} elif url.endswith("/api/projects"): payload = {{"id": "project-1"}} elif url.endswith("/api/projects/project-1/workflows"): pathlib.Path({str(captured_workflow)!r}).write_text(body, encoding="utf-8") payload = {{"id": "workflow-1"}} elif url.endswith("/api/workflows/workflow-1/run"): payload = {{"id": "run-1"}} elif url.endswith("/api/runs/run-1/logs"): payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}} elif url.endswith("/api/runs/run-1/artifacts"): payload = {{"items": []}} elif url.endswith("/api/runs/run-1"): payload = {{"status": "failed"}} else: raise SystemExit(f"unexpected URL: {{url}}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) curl.chmod(0o755) env = os.environ.copy() env.update( { "DEPLOY_ENV": "poc", "DROBOTICFLOW_ACCEPTANCE_EMAIL": "operator@example.test", "DROBOTICFLOW_ACCEPTANCE_PASSWORD": "secret", "K8S_NAMESPACE": "droboticflow", "PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}", "POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "false", "POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", } ) env.pop("API_BASE_URL", None) env.pop("BASE_URL", None) result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) assert result.returncode == 0, result.stderr + result.stdout assert "run status: failed" in result.stdout workflow = json.loads(captured_workflow.read_text(encoding="utf-8")) source_node = next(node for node in workflow["nodes"] if node["id"] == "source-1") assert source_node["data"]["gpu"] == "1" def test_poc_gpu_workflow_acceptance_uses_token_endpoint_and_worker_port_forward(tmp_path): captured_acceptance = tmp_path / "acceptance.json" run_status = tmp_path / "run-status.txt" stub_bin = tmp_path / "bin" stub_bin.mkdir() kubectl = stub_bin / "kubectl" kubectl.write_text( """#!/usr/bin/env python3 import json import sys import time args = sys.argv[1:] if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]: print(json.dumps({ "status": { "loadBalancer": { "ingress": [ {"ip": "192.168.0.74"}, {"ip": "120.48.164.19"} ] } } })) raise SystemExit(0) if args == ["-n", "droboticflow", "get", "secret", "droboticflow-auth-secret", "-o", "json"]: print(json.dumps({ "data": { "DROBOTICFLOW_API_TOKEN": "c2hhcmVkLXRva2Vu" } })) raise SystemExit(0) if len(args) >= 5 and args[:4] == ["-n", "droboticflow", "port-forward", "service/droboticflow-worker"]: print("Forwarding from 127.0.0.1 -> 3002", flush=True) time.sleep(30) raise SystemExit(0) print("unexpected kubectl call: " + " ".join(args), file=sys.stderr) raise SystemExit(97) """, encoding="utf-8", ) kubectl.chmod(0o755) curl = stub_bin / "curl" curl.write_text( f"""#!/usr/bin/env python3 import json import pathlib import sys status_file = pathlib.Path({str(run_status)!r}) captured_acceptance = pathlib.Path({str(captured_acceptance)!r}) args = sys.argv[1:] url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "") headers = [args[index + 1] for index, arg in enumerate(args[:-1]) if arg == "-H"] body = "" for index, arg in enumerate(args): if arg == "-d" and index + 1 < len(args): body = args[index + 1] if url.startswith("http://120.48.164.19") and "Authorization: Bearer shared-token" not in headers: raise SystemExit("expected bearer token for API call") if url.startswith("http://") and "120.48.164.19" not in url and "127.0.0.1" not in url: raise SystemExit(f"expected public load balancer or worker port-forward URL, got {{url}}") if url.endswith("/api/health"): payload = {{"status": "ok"}} elif url.endswith("/api/acceptance/poc-gpu-workflow"): captured_acceptance.write_text(body, encoding="utf-8") status_file.write_text("queued", encoding="utf-8") payload = {{ "project": {{"id": "project-token", "name": "POC GPU Workflow Acceptance"}}, "workflow": {{"id": "workflow-token", "name": "POC GPU Unschedulable Acceptance Workflow"}}, "run": {{"id": "run-token", "status": "queued"}} }} elif url.endswith("/worker-api/health"): payload = {{"status": "ok", "service": "droboticflow-worker"}} elif url.endswith("/worker-api/process-once"): status_file.write_text("failed", encoding="utf-8") payload = {{"run": {{"id": "run-token", "status": "failed"}}}} elif url.endswith("/api/runs/run-token/logs"): payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}} elif url.endswith("/api/runs/run-token"): payload = {{ "id": "run-token", "status": status_file.read_text(encoding="utf-8").strip(), "resourceSeries": [{{"timestamp": "2026-06-28T00:00:00Z", "cpu": 1, "memory": 64, "gpu": 3, "gpuMemoryUsedMiB": 1024, "gpuMemoryTotalMiB": 81920}}], }} else: raise SystemExit(f"unexpected URL: {{url}}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) curl.chmod(0o755) env = os.environ.copy() env.update( { "DEPLOY_ENV": "poc", "K8S_NAMESPACE": "droboticflow", "PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}", "POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "false", "POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", } ) env.pop("API_BASE_URL", None) env.pop("BASE_URL", None) env.pop("DROBOTICFLOW_API_TOKEN", None) env.pop("DROBOTICFLOW_ACCEPTANCE_EMAIL", None) env.pop("DROBOTICFLOW_ACCEPTANCE_PASSWORD", None) result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) assert result.returncode == 0, result.stderr + result.stdout assert "created POC GPU acceptance project=project-token workflow=workflow-token run=run-token" in result.stdout assert "worker port-forward:" in result.stdout assert "run status: failed" in result.stdout body = json.loads(captured_acceptance.read_text(encoding="utf-8")) assert body == {"sourcePath": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform"} def test_poc_gpu_workflow_acceptance_can_expect_completed_gpu_run(tmp_path): captured_acceptance = tmp_path / "acceptance.json" run_status = tmp_path / "run-status.txt" stub_bin = tmp_path / "bin" stub_bin.mkdir() kubectl = stub_bin / "kubectl" kubectl.write_text( """#!/usr/bin/env python3 import json import sys import time args = sys.argv[1:] if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]: print(json.dumps({"status": {"loadBalancer": {"ingress": [{"ip": "120.48.164.19"}]}}})) raise SystemExit(0) if args == ["-n", "droboticflow", "get", "secret", "droboticflow-auth-secret", "-o", "json"]: print(json.dumps({"data": {"DROBOTICFLOW_API_TOKEN": "c2hhcmVkLXRva2Vu"}})) raise SystemExit(0) if len(args) >= 5 and args[:4] == ["-n", "droboticflow", "port-forward", "service/droboticflow-worker"]: print("Forwarding from 127.0.0.1 -> 3002", flush=True) time.sleep(30) raise SystemExit(0) print("unexpected kubectl call: " + " ".join(args), file=sys.stderr) raise SystemExit(97) """, encoding="utf-8", ) kubectl.chmod(0o755) curl = stub_bin / "curl" curl.write_text( f"""#!/usr/bin/env python3 import json import pathlib import sys status_file = pathlib.Path({str(run_status)!r}) captured_acceptance = pathlib.Path({str(captured_acceptance)!r}) args = sys.argv[1:] url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "") body = "" for index, arg in enumerate(args): if arg == "-d" and index + 1 < len(args): body = args[index + 1] if url.endswith("/api/health"): payload = {{"status": "ok"}} elif url.endswith("/api/acceptance/poc-gpu-workflow"): captured_acceptance.write_text(body, encoding="utf-8") status_file.write_text("queued", encoding="utf-8") payload = {{"project": {{"id": "project-token"}}, "workflow": {{"id": "workflow-token"}}, "run": {{"id": "run-token", "status": "queued"}}}} elif url.endswith("/worker-api/health"): payload = {{"status": "ok"}} elif url.endswith("/worker-api/process-once"): status_file.write_text("completed", encoding="utf-8") payload = {{"run": {{"id": "run-token", "status": "completed"}}}} elif url.endswith("/api/runs/run-token/logs"): payload = {{"items": [{{"message": "GPU operator completed"}}]}} elif url.endswith("/api/runs/run-token/artifacts"): payload = {{"items": [{{"name": "manifest.json", "type": "json", "size": 128}}]}} elif url.endswith("/api/runs/run-token"): payload = {{ "id": "run-token", "status": status_file.read_text(encoding="utf-8").strip(), "resourceSeries": [{{"timestamp": "2026-06-28T00:00:00Z", "cpu": 1, "memory": 64, "gpu": 3, "gpuMemoryUsedMiB": 1024, "gpuMemoryTotalMiB": 81920}}], }} else: raise SystemExit(f"unexpected URL: {{url}}") print(json.dumps(payload, separators=(",", ":"))) """, encoding="utf-8", ) curl.chmod(0o755) env = os.environ.copy() env.update( { "DEPLOY_ENV": "poc", "K8S_NAMESPACE": "droboticflow", "PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}", "POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "true", "POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform", "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", } ) env.pop("API_BASE_URL", None) env.pop("BASE_URL", None) env.pop("DROBOTICFLOW_API_TOKEN", None) env.pop("DROBOTICFLOW_ACCEPTANCE_EMAIL", None) env.pop("DROBOTICFLOW_ACCEPTANCE_PASSWORD", None) result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) assert result.returncode == 0, result.stderr + result.stdout assert "run status: completed" in result.stdout assert "artifacts summary:" in result.stdout body = json.loads(captured_acceptance.read_text(encoding="utf-8")) assert body == {"sourcePath": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform"}