import os import pathlib import subprocess import json ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "gpu-acceptance-test.sh" def default_nodes_payload() -> dict: return { "items": [ { "metadata": {"name": "gpu-node-1", "labels": {"accelerator": "nvidia"}}, "status": {"allocatable": {"nvidia.com/gpu": "2"}}, } ] } def write_kubectl_stub( tmp_path: pathlib.Path, capture_manifest: pathlib.Path | None = None, nodes_payload: dict | None = None, ) -> pathlib.Path: stub_bin = tmp_path / "bin" stub_bin.mkdir() kubectl = stub_bin / "kubectl" capture_line = "capture_manifest = None\n" if capture_manifest is not None: capture_line = f"capture_manifest = {str(capture_manifest)!r}\n" nodes_payload = nodes_payload or default_nodes_payload() kubectl.write_text( "#!/usr/bin/env python3\n" "import json\n" "import shutil\n" "import sys\n" f"{capture_line}" f"nodes_payload = {json.dumps(nodes_payload)!r}\n" "args = sys.argv[1:]\n" "if args[:2] == ['get', 'namespace']:\n" " raise SystemExit(0)\n" "if args == ['get', 'nodes', '-o', 'json']:\n" " print(json.dumps(json.loads(nodes_payload)))\n" " raise SystemExit(0)\n" "if args[:2] == ['apply', '-f']:\n" " if capture_manifest:\n" " shutil.copyfile(args[2], capture_manifest)\n" " print('job.batch/droboticflow-gpu-acceptance-test created')\n" " raise SystemExit(0)\n" "if len(args) >= 2 and args[0] == '-n':\n" " command = args[2:]\n" " if command[:1] == ['wait']:\n" " print('job.batch/droboticflow-gpu-acceptance-test condition met')\n" " raise SystemExit(0)\n" " if command[:1] == ['logs']:\n" " print('droboticflow-gpu-acceptance-ok')\n" " raise SystemExit(0)\n" " if command[:1] == ['delete']:\n" " raise SystemExit(0)\n" "print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n" "raise SystemExit(97)\n", encoding="utf-8", ) kubectl.chmod(0o755) return stub_bin def run_gpu_acceptance( tmp_path: pathlib.Path, extra_env: dict[str, str], capture_manifest: pathlib.Path | None = None, default_name: bool = True, nodes_payload: dict | None = None, ) -> subprocess.CompletedProcess: stub_bin = write_kubectl_stub(tmp_path, capture_manifest=capture_manifest, nodes_payload=nodes_payload) env = os.environ.copy() env.update( { "PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}", "K8S_NAMESPACE": "droboticflow", } ) if default_name: env["GPU_ACCEPTANCE_NAME"] = "droboticflow-gpu-acceptance-test" env.update(extra_env) return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) def assert_validation_failure(result: subprocess.CompletedProcess, message: str) -> None: assert result.returncode == 1, result.stderr + result.stdout assert message in result.stderr assert "unexpected kubectl call" not in result.stderr def test_gpu_acceptance_rejects_invalid_gpu_count_before_kubectl(tmp_path): result = run_gpu_acceptance(tmp_path, {"GPU_RESOURCE_COUNT": "0"}) assert_validation_failure(result, "GPU_RESOURCE_COUNT must be a positive integer") def test_gpu_acceptance_rejects_invalid_namespace_before_kubectl(tmp_path): result = run_gpu_acceptance(tmp_path, {"K8S_NAMESPACE": "droboticflow\nother"}) assert_validation_failure(result, "K8S_NAMESPACE must be a lowercase DNS label") def test_gpu_acceptance_rejects_invalid_timeout_before_kubectl(tmp_path): result = run_gpu_acceptance(tmp_path, {"GPU_ACCEPTANCE_TIMEOUT": "five-minutes"}) assert_validation_failure(result, "GPU_ACCEPTANCE_TIMEOUT must be a positive Go-style duration") def test_gpu_acceptance_rejects_image_reference_with_whitespace_before_kubectl(tmp_path): result = run_gpu_acceptance(tmp_path, {"GPU_ACCEPTANCE_IMAGE": "nvidia/cuda:12.4 test"}) assert_validation_failure(result, "GPU_ACCEPTANCE_IMAGE must be a non-empty container image reference without whitespace") def test_gpu_acceptance_rejects_invalid_selector_before_kubectl(tmp_path): result = run_gpu_acceptance(tmp_path, {"OPERATOR_GPU_NODE_SELECTOR": "accelerator"}) assert_validation_failure(result, "invalid OPERATOR_GPU_NODE_SELECTOR item") def test_gpu_acceptance_rejects_invalid_tolerations_before_kubectl(tmp_path): result = run_gpu_acceptance(tmp_path, {"OPERATOR_GPU_TOLERATIONS_JSON": '{"key":"gpu"}'}) assert_validation_failure(result, "OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array") def test_gpu_acceptance_renders_valid_scheduling_manifest(tmp_path): manifest = tmp_path / "gpu-acceptance.yaml" result = run_gpu_acceptance( tmp_path, { "OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia", "OPERATOR_GPU_TOLERATIONS_JSON": '[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]', "OPERATOR_GPU_RUNTIME_CLASS": "nvidia", }, capture_manifest=manifest, ) assert result.returncode == 0, result.stderr + result.stdout assert "droboticflow-gpu-acceptance-ok" in result.stdout rendered = manifest.read_text(encoding="utf-8") assert 'name: "droboticflow-gpu-acceptance-test"' in rendered assert 'namespace: "droboticflow"' in rendered assert 'image: "nvidia/cuda:12.4.1-base-ubuntu22.04"' in rendered assert '"nvidia.com/gpu": "1"' in rendered assert '"accelerator": "nvidia"' in rendered assert 'key: "nvidia.com/gpu"' in rendered assert 'operator: "Exists"' in rendered assert 'effect: "NoSchedule"' in rendered assert 'runtimeClassName: "nvidia"' in rendered def test_gpu_acceptance_rejects_selector_mismatch_before_creating_job(tmp_path): nodes_payload = { "items": [ { "metadata": {"name": "gpu-node-1", "labels": {"accelerator": "amd"}}, "status": {"allocatable": {"nvidia.com/gpu": "4"}}, } ] } result = run_gpu_acceptance( tmp_path, {"OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia"}, nodes_payload=nodes_payload, ) assert result.returncode == 1 assert "selector accelerator=nvidia" in result.stderr assert "selectorMatched=false" in result.stderr assert "unexpected kubectl call" not in result.stderr def test_gpu_acceptance_rejects_unschedulable_gpu_node_before_creating_job(tmp_path): nodes_payload = { "items": [ { "metadata": {"name": "gpu-node-1", "labels": {"accelerator": "nvidia"}}, "spec": {"unschedulable": True}, "status": {"allocatable": {"nvidia.com/gpu": "4"}}, } ] } result = run_gpu_acceptance( tmp_path, {"OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia"}, nodes_payload=nodes_payload, ) assert result.returncode == 1 assert "unschedulable=true" in result.stderr assert "selectorMatched=true" in result.stderr assert "unexpected kubectl call" not in result.stderr def test_gpu_acceptance_loads_deploy_env_before_defaults(tmp_path): env_file = ROOT / ".env.poc.local" original = env_file.read_text(encoding="utf-8") if env_file.exists() else None manifest = tmp_path / "gpu-acceptance-env-loaded.yaml" try: env_file.write_text('export GPU_ACCEPTANCE_NAME="loaded-gpu-acceptance"\n', encoding="utf-8") result = run_gpu_acceptance( tmp_path, {"DEPLOY_ENV": "poc"}, capture_manifest=manifest, default_name=False, ) finally: if original is None: env_file.unlink(missing_ok=True) else: env_file.write_text(original, encoding="utf-8") assert result.returncode == 0, result.stderr + result.stdout rendered = manifest.read_text(encoding="utf-8") assert 'name: "loaded-gpu-acceptance"' in rendered