import json import os import pathlib import subprocess ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "poc-gpu-capacity-acceptance.sh" def write_kubectl_stub(tmp_path: pathlib.Path, capacity_payload: dict, capture_manifest: pathlib.Path) -> pathlib.Path: stub_bin = tmp_path / "bin" stub_bin.mkdir() kubectl = stub_bin / "kubectl" kubectl.write_text( f"""#!/usr/bin/env python3 import json import pathlib import shutil import sys args = sys.argv[1:] capture_manifest = pathlib.Path({str(capture_manifest)!r}) capacity_payload = {json.dumps(capacity_payload)!r} if args == ["get", "namespace", "droboticflow"]: raise SystemExit(0) if args == ["-n", "droboticflow", "get", "configmap", "droboticflow-config", "-o", "jsonpath={{.data.OPERATOR_RUNNER_IMAGE}}"]: print("ccr.example.test/dataflow-poc/droboticflow-operator-runner:deployed", end="") raise SystemExit(0) if args == ["-n", "droboticflow", "get", "configmap", "droboticflow-config", "-o", "jsonpath={{.data.K8S_IMAGE_PULL_SECRET_NAME}}"]: print("droboticflow-registry", end="") raise SystemExit(0) if args[:2] == ["apply", "-f"]: shutil.copyfile(args[2], capture_manifest) print("pod/droboticflow-gpu-capacity-test created") raise SystemExit(0) if args == ["-n", "droboticflow", "get", "pod", "droboticflow-gpu-capacity-test", "-o", "jsonpath={{.status.phase}}"]: print("Succeeded", end="") raise SystemExit(0) if args == ["-n", "droboticflow", "logs", "droboticflow-gpu-capacity-test"]: print(json.dumps(json.loads(capacity_payload), separators=(",", ":"))) raise SystemExit(0) if args[:4] == ["-n", "droboticflow", "delete", "pod"]: raise SystemExit(0) print("unexpected kubectl call: " + " ".join(args), file=sys.stderr) raise SystemExit(97) """, encoding="utf-8", ) kubectl.chmod(0o755) return stub_bin def run_capacity_acceptance( tmp_path: pathlib.Path, capacity_payload: dict, extra_env: dict[str, str] | None = None, ) -> tuple[subprocess.CompletedProcess, pathlib.Path]: manifest = tmp_path / "capacity-pod.yaml" stub_bin = write_kubectl_stub(tmp_path, capacity_payload, manifest) env = os.environ.copy() env.update( { "DEPLOY_ENV": "poc", "K8S_NAMESPACE": "droboticflow", "PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}", "POC_GPU_CAPACITY_NAME": "droboticflow-gpu-capacity-test", "POC_GPU_CAPACITY_TIMEOUT_SECONDS": "1", "POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS": "1", } ) env.pop("OPERATOR_RUNNER_IMAGE", None) env.pop("POC_GPU_CAPACITY_IMAGE", None) if extra_env: env.update(extra_env) result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) return result, manifest def cpu_only_capacity_payload() -> dict: return { "gpu": { "resourceName": "nvidia.com/gpu", "available": False, "schedulableAllocatable": 0, "maxAllocatable": 0, "schedulableNodeCount": 0, "nodes": [ {"name": "192.168.80.249", "allocatable": 0, "selectorMatched": True}, {"name": "192.168.81.47", "allocatable": 0, "selectorMatched": True}, ], } } def available_capacity_payload() -> dict: return { "gpu": { "resourceName": "nvidia.com/gpu", "available": True, "schedulableAllocatable": 4, "maxAllocatable": 4, "schedulableNodeCount": 1, "nodes": [{"name": "gpu-node-1", "allocatable": 4, "selectorMatched": True}], } } def test_poc_gpu_capacity_acceptance_reads_deployed_worker_capacity(tmp_path): result, manifest = run_capacity_acceptance(tmp_path, available_capacity_payload()) assert result.returncode == 0, result.stderr + result.stdout assert "gpu.available=true resource=nvidia.com/gpu schedulableAllocatable=4 maxAllocatable=4 nodes=1" in result.stdout assert "node gpu-node-1 nvidia.com/gpu=4 selectorMatched=true" in result.stdout rendered = manifest.read_text(encoding="utf-8") assert 'name: "droboticflow-gpu-capacity-test"' in rendered assert 'namespace: "droboticflow"' in rendered assert 'image: "ccr.example.test/dataflow-poc/droboticflow-operator-runner:deployed"' in rendered assert 'name: "droboticflow-registry"' in rendered assert 'value: "http://droboticflow-worker:3002/worker-api/capacity"' in rendered def test_poc_gpu_capacity_acceptance_can_expect_available_gpu(tmp_path): payload = { "gpu": { "resourceName": "nvidia.com/gpu", "available": True, "schedulableAllocatable": 4, "maxAllocatable": 4, "schedulableNodeCount": 1, "nodes": [{"name": "gpu-node-1", "allocatable": 4, "selectorMatched": True}], } } result, _ = run_capacity_acceptance( tmp_path, payload, {"POC_GPU_CAPACITY_EXPECT_AVAILABLE": "true"}, ) assert result.returncode == 0, result.stderr + result.stdout assert "gpu.available=true resource=nvidia.com/gpu schedulableAllocatable=4 maxAllocatable=4 nodes=1" in result.stdout def test_poc_gpu_capacity_acceptance_fails_when_cpu_only_expected_but_gpu_available(tmp_path): payload = { "gpu": { "resourceName": "nvidia.com/gpu", "available": True, "schedulableAllocatable": 1, "maxAllocatable": 1, "nodes": [{"name": "gpu-node-1", "allocatable": 1, "selectorMatched": True}], } } result, _ = run_capacity_acceptance(tmp_path, payload, {"POC_GPU_CAPACITY_EXPECT_AVAILABLE": "false"}) assert result.returncode == 1 assert "expected CPU-only POC GPU capacity" in result.stderr def test_poc_gpu_capacity_acceptance_rejects_invalid_inputs_before_kubectl(tmp_path): result, _ = run_capacity_acceptance( tmp_path, cpu_only_capacity_payload(), {"POC_GPU_CAPACITY_NAME": "bad_name"}, ) assert result.returncode == 1 assert "POC_GPU_CAPACITY_NAME must be a lowercase DNS label" in result.stderr assert "unexpected kubectl call" not in result.stderr