droboticflow/tests/backend/gpu_acceptance_script_test.py
2026-06-23 06:38:41 +08:00

136 lines
5.4 KiB
Python

import os
import pathlib
import subprocess
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "gpu-acceptance-test.sh"
def write_kubectl_stub(tmp_path: pathlib.Path, capture_manifest: pathlib.Path | None = None) -> pathlib.Path:
stub_bin = tmp_path / "bin"
stub_bin.mkdir()
kubectl = stub_bin / "kubectl"
capture_line = "capture_manifest = None\n"
if capture_manifest is not None:
capture_line = f"capture_manifest = {str(capture_manifest)!r}\n"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import json\n"
"import shutil\n"
"import sys\n"
f"{capture_line}"
"args = sys.argv[1:]\n"
"if args[:2] == ['get', 'namespace']:\n"
" raise SystemExit(0)\n"
"if args == ['get', 'nodes', '-o', 'json']:\n"
" print(json.dumps({'items': [{'metadata': {'name': 'gpu-node-1'}, 'status': {'allocatable': {'nvidia.com/gpu': '2'}}}]}))\n"
" raise SystemExit(0)\n"
"if args[:2] == ['apply', '-f']:\n"
" if capture_manifest:\n"
" shutil.copyfile(args[2], capture_manifest)\n"
" print('job.batch/droboticflow-gpu-acceptance-test created')\n"
" raise SystemExit(0)\n"
"if len(args) >= 2 and args[0] == '-n':\n"
" command = args[2:]\n"
" if command[:1] == ['wait']:\n"
" print('job.batch/droboticflow-gpu-acceptance-test condition met')\n"
" raise SystemExit(0)\n"
" if command[:1] == ['logs']:\n"
" print('droboticflow-gpu-acceptance-ok')\n"
" raise SystemExit(0)\n"
" if command[:1] == ['delete']:\n"
" raise SystemExit(0)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return stub_bin
def run_gpu_acceptance(
tmp_path: pathlib.Path,
extra_env: dict[str, str],
capture_manifest: pathlib.Path | None = None,
) -> subprocess.CompletedProcess:
stub_bin = write_kubectl_stub(tmp_path, capture_manifest=capture_manifest)
env = os.environ.copy()
env.update(
{
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
"K8S_NAMESPACE": "droboticflow",
"GPU_ACCEPTANCE_NAME": "droboticflow-gpu-acceptance-test",
}
)
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def assert_validation_failure(result: subprocess.CompletedProcess, message: str) -> None:
assert result.returncode == 1, result.stderr + result.stdout
assert message in result.stderr
assert "unexpected kubectl call" not in result.stderr
def test_gpu_acceptance_rejects_invalid_gpu_count_before_kubectl(tmp_path):
result = run_gpu_acceptance(tmp_path, {"GPU_RESOURCE_COUNT": "0"})
assert_validation_failure(result, "GPU_RESOURCE_COUNT must be a positive integer")
def test_gpu_acceptance_rejects_invalid_namespace_before_kubectl(tmp_path):
result = run_gpu_acceptance(tmp_path, {"K8S_NAMESPACE": "droboticflow\nother"})
assert_validation_failure(result, "K8S_NAMESPACE must be a lowercase DNS label")
def test_gpu_acceptance_rejects_invalid_timeout_before_kubectl(tmp_path):
result = run_gpu_acceptance(tmp_path, {"GPU_ACCEPTANCE_TIMEOUT": "five-minutes"})
assert_validation_failure(result, "GPU_ACCEPTANCE_TIMEOUT must be a positive Go-style duration")
def test_gpu_acceptance_rejects_image_reference_with_whitespace_before_kubectl(tmp_path):
result = run_gpu_acceptance(tmp_path, {"GPU_ACCEPTANCE_IMAGE": "nvidia/cuda:12.4 test"})
assert_validation_failure(result, "GPU_ACCEPTANCE_IMAGE must be a non-empty container image reference without whitespace")
def test_gpu_acceptance_rejects_invalid_selector_before_kubectl(tmp_path):
result = run_gpu_acceptance(tmp_path, {"OPERATOR_GPU_NODE_SELECTOR": "accelerator"})
assert_validation_failure(result, "invalid OPERATOR_GPU_NODE_SELECTOR item")
def test_gpu_acceptance_rejects_invalid_tolerations_before_kubectl(tmp_path):
result = run_gpu_acceptance(tmp_path, {"OPERATOR_GPU_TOLERATIONS_JSON": '{"key":"gpu"}'})
assert_validation_failure(result, "OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array")
def test_gpu_acceptance_renders_valid_scheduling_manifest(tmp_path):
manifest = tmp_path / "gpu-acceptance.yaml"
result = run_gpu_acceptance(
tmp_path,
{
"OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia",
"OPERATOR_GPU_TOLERATIONS_JSON": '[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]',
"OPERATOR_GPU_RUNTIME_CLASS": "nvidia",
},
capture_manifest=manifest,
)
assert result.returncode == 0, result.stderr + result.stdout
assert "droboticflow-gpu-acceptance-ok" in result.stdout
rendered = manifest.read_text(encoding="utf-8")
assert 'name: "droboticflow-gpu-acceptance-test"' in rendered
assert 'namespace: "droboticflow"' in rendered
assert 'image: "nvidia/cuda:12.4.1-base-ubuntu22.04"' in rendered
assert '"nvidia.com/gpu": "1"' in rendered
assert '"accelerator": "nvidia"' in rendered
assert 'key: "nvidia.com/gpu"' in rendered
assert 'operator: "Exists"' in rendered
assert 'effect: "NoSchedule"' in rendered
assert 'runtimeClassName: "nvidia"' in rendered