forked from longtao.wu/droboticflow
136 lines
5.4 KiB
Python
136 lines
5.4 KiB
Python
import os
|
|
import pathlib
|
|
import subprocess
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "gpu-acceptance-test.sh"
|
|
|
|
|
|
def write_kubectl_stub(tmp_path: pathlib.Path, capture_manifest: pathlib.Path | None = None) -> pathlib.Path:
|
|
stub_bin = tmp_path / "bin"
|
|
stub_bin.mkdir()
|
|
kubectl = stub_bin / "kubectl"
|
|
capture_line = "capture_manifest = None\n"
|
|
if capture_manifest is not None:
|
|
capture_line = f"capture_manifest = {str(capture_manifest)!r}\n"
|
|
kubectl.write_text(
|
|
"#!/usr/bin/env python3\n"
|
|
"import json\n"
|
|
"import shutil\n"
|
|
"import sys\n"
|
|
f"{capture_line}"
|
|
"args = sys.argv[1:]\n"
|
|
"if args[:2] == ['get', 'namespace']:\n"
|
|
" raise SystemExit(0)\n"
|
|
"if args == ['get', 'nodes', '-o', 'json']:\n"
|
|
" print(json.dumps({'items': [{'metadata': {'name': 'gpu-node-1'}, 'status': {'allocatable': {'nvidia.com/gpu': '2'}}}]}))\n"
|
|
" raise SystemExit(0)\n"
|
|
"if args[:2] == ['apply', '-f']:\n"
|
|
" if capture_manifest:\n"
|
|
" shutil.copyfile(args[2], capture_manifest)\n"
|
|
" print('job.batch/droboticflow-gpu-acceptance-test created')\n"
|
|
" raise SystemExit(0)\n"
|
|
"if len(args) >= 2 and args[0] == '-n':\n"
|
|
" command = args[2:]\n"
|
|
" if command[:1] == ['wait']:\n"
|
|
" print('job.batch/droboticflow-gpu-acceptance-test condition met')\n"
|
|
" raise SystemExit(0)\n"
|
|
" if command[:1] == ['logs']:\n"
|
|
" print('droboticflow-gpu-acceptance-ok')\n"
|
|
" raise SystemExit(0)\n"
|
|
" if command[:1] == ['delete']:\n"
|
|
" raise SystemExit(0)\n"
|
|
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
|
|
"raise SystemExit(97)\n",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
return stub_bin
|
|
|
|
|
|
def run_gpu_acceptance(
|
|
tmp_path: pathlib.Path,
|
|
extra_env: dict[str, str],
|
|
capture_manifest: pathlib.Path | None = None,
|
|
) -> subprocess.CompletedProcess:
|
|
stub_bin = write_kubectl_stub(tmp_path, capture_manifest=capture_manifest)
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"GPU_ACCEPTANCE_NAME": "droboticflow-gpu-acceptance-test",
|
|
}
|
|
)
|
|
env.update(extra_env)
|
|
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
|
|
|
|
def assert_validation_failure(result: subprocess.CompletedProcess, message: str) -> None:
|
|
assert result.returncode == 1, result.stderr + result.stdout
|
|
assert message in result.stderr
|
|
assert "unexpected kubectl call" not in result.stderr
|
|
|
|
|
|
def test_gpu_acceptance_rejects_invalid_gpu_count_before_kubectl(tmp_path):
|
|
result = run_gpu_acceptance(tmp_path, {"GPU_RESOURCE_COUNT": "0"})
|
|
|
|
assert_validation_failure(result, "GPU_RESOURCE_COUNT must be a positive integer")
|
|
|
|
|
|
def test_gpu_acceptance_rejects_invalid_namespace_before_kubectl(tmp_path):
|
|
result = run_gpu_acceptance(tmp_path, {"K8S_NAMESPACE": "droboticflow\nother"})
|
|
|
|
assert_validation_failure(result, "K8S_NAMESPACE must be a lowercase DNS label")
|
|
|
|
|
|
def test_gpu_acceptance_rejects_invalid_timeout_before_kubectl(tmp_path):
|
|
result = run_gpu_acceptance(tmp_path, {"GPU_ACCEPTANCE_TIMEOUT": "five-minutes"})
|
|
|
|
assert_validation_failure(result, "GPU_ACCEPTANCE_TIMEOUT must be a positive Go-style duration")
|
|
|
|
|
|
def test_gpu_acceptance_rejects_image_reference_with_whitespace_before_kubectl(tmp_path):
|
|
result = run_gpu_acceptance(tmp_path, {"GPU_ACCEPTANCE_IMAGE": "nvidia/cuda:12.4 test"})
|
|
|
|
assert_validation_failure(result, "GPU_ACCEPTANCE_IMAGE must be a non-empty container image reference without whitespace")
|
|
|
|
|
|
def test_gpu_acceptance_rejects_invalid_selector_before_kubectl(tmp_path):
|
|
result = run_gpu_acceptance(tmp_path, {"OPERATOR_GPU_NODE_SELECTOR": "accelerator"})
|
|
|
|
assert_validation_failure(result, "invalid OPERATOR_GPU_NODE_SELECTOR item")
|
|
|
|
|
|
def test_gpu_acceptance_rejects_invalid_tolerations_before_kubectl(tmp_path):
|
|
result = run_gpu_acceptance(tmp_path, {"OPERATOR_GPU_TOLERATIONS_JSON": '{"key":"gpu"}'})
|
|
|
|
assert_validation_failure(result, "OPERATOR_GPU_TOLERATIONS_JSON must be a JSON array")
|
|
|
|
|
|
def test_gpu_acceptance_renders_valid_scheduling_manifest(tmp_path):
|
|
manifest = tmp_path / "gpu-acceptance.yaml"
|
|
result = run_gpu_acceptance(
|
|
tmp_path,
|
|
{
|
|
"OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia",
|
|
"OPERATOR_GPU_TOLERATIONS_JSON": '[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]',
|
|
"OPERATOR_GPU_RUNTIME_CLASS": "nvidia",
|
|
},
|
|
capture_manifest=manifest,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "droboticflow-gpu-acceptance-ok" in result.stdout
|
|
rendered = manifest.read_text(encoding="utf-8")
|
|
assert 'name: "droboticflow-gpu-acceptance-test"' in rendered
|
|
assert 'namespace: "droboticflow"' in rendered
|
|
assert 'image: "nvidia/cuda:12.4.1-base-ubuntu22.04"' in rendered
|
|
assert '"nvidia.com/gpu": "1"' in rendered
|
|
assert '"accelerator": "nvidia"' in rendered
|
|
assert 'key: "nvidia.com/gpu"' in rendered
|
|
assert 'operator: "Exists"' in rendered
|
|
assert 'effect: "NoSchedule"' in rendered
|
|
assert 'runtimeClassName: "nvidia"' in rendered
|