droboticflow/tests/backend/poc_gpu_capacity_acceptance_script_test.py
2026-07-02 12:54:48 +08:00

171 lines
6.2 KiB
Python

import json
import os
import pathlib
import subprocess
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "poc-gpu-capacity-acceptance.sh"
def write_kubectl_stub(tmp_path: pathlib.Path, capacity_payload: dict, capture_manifest: pathlib.Path) -> pathlib.Path:
stub_bin = tmp_path / "bin"
stub_bin.mkdir()
kubectl = stub_bin / "kubectl"
kubectl.write_text(
f"""#!/usr/bin/env python3
import json
import pathlib
import shutil
import sys
args = sys.argv[1:]
capture_manifest = pathlib.Path({str(capture_manifest)!r})
capacity_payload = {json.dumps(capacity_payload)!r}
if args == ["get", "namespace", "droboticflow"]:
raise SystemExit(0)
if args == ["-n", "droboticflow", "get", "configmap", "droboticflow-config", "-o", "jsonpath={{.data.OPERATOR_RUNNER_IMAGE}}"]:
print("ccr.example.test/dataflow-poc/droboticflow-operator-runner:deployed", end="")
raise SystemExit(0)
if args == ["-n", "droboticflow", "get", "configmap", "droboticflow-config", "-o", "jsonpath={{.data.K8S_IMAGE_PULL_SECRET_NAME}}"]:
print("droboticflow-registry", end="")
raise SystemExit(0)
if args[:2] == ["apply", "-f"]:
shutil.copyfile(args[2], capture_manifest)
print("pod/droboticflow-gpu-capacity-test created")
raise SystemExit(0)
if args == ["-n", "droboticflow", "get", "pod", "droboticflow-gpu-capacity-test", "-o", "jsonpath={{.status.phase}}"]:
print("Succeeded", end="")
raise SystemExit(0)
if args == ["-n", "droboticflow", "logs", "droboticflow-gpu-capacity-test"]:
print(json.dumps(json.loads(capacity_payload), separators=(",", ":")))
raise SystemExit(0)
if args[:4] == ["-n", "droboticflow", "delete", "pod"]:
raise SystemExit(0)
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
raise SystemExit(97)
""",
encoding="utf-8",
)
kubectl.chmod(0o755)
return stub_bin
def run_capacity_acceptance(
tmp_path: pathlib.Path,
capacity_payload: dict,
extra_env: dict[str, str] | None = None,
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
manifest = tmp_path / "capacity-pod.yaml"
stub_bin = write_kubectl_stub(tmp_path, capacity_payload, manifest)
env = os.environ.copy()
env.update(
{
"DEPLOY_ENV": "poc",
"K8S_NAMESPACE": "droboticflow",
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
"POC_GPU_CAPACITY_NAME": "droboticflow-gpu-capacity-test",
"POC_GPU_CAPACITY_TIMEOUT_SECONDS": "1",
"POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS": "1",
}
)
env.pop("OPERATOR_RUNNER_IMAGE", None)
env.pop("POC_GPU_CAPACITY_IMAGE", None)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, manifest
def cpu_only_capacity_payload() -> dict:
return {
"gpu": {
"resourceName": "nvidia.com/gpu",
"available": False,
"schedulableAllocatable": 0,
"maxAllocatable": 0,
"schedulableNodeCount": 0,
"nodes": [
{"name": "192.168.80.249", "allocatable": 0, "selectorMatched": True},
{"name": "192.168.81.47", "allocatable": 0, "selectorMatched": True},
],
}
}
def available_capacity_payload() -> dict:
return {
"gpu": {
"resourceName": "nvidia.com/gpu",
"available": True,
"schedulableAllocatable": 4,
"maxAllocatable": 4,
"schedulableNodeCount": 1,
"nodes": [{"name": "gpu-node-1", "allocatable": 4, "selectorMatched": True}],
}
}
def test_poc_gpu_capacity_acceptance_reads_deployed_worker_capacity(tmp_path):
result, manifest = run_capacity_acceptance(tmp_path, available_capacity_payload())
assert result.returncode == 0, result.stderr + result.stdout
assert "gpu.available=true resource=nvidia.com/gpu schedulableAllocatable=4 maxAllocatable=4 nodes=1" in result.stdout
assert "node gpu-node-1 nvidia.com/gpu=4 selectorMatched=true" in result.stdout
rendered = manifest.read_text(encoding="utf-8")
assert 'name: "droboticflow-gpu-capacity-test"' in rendered
assert 'namespace: "droboticflow"' in rendered
assert 'image: "ccr.example.test/dataflow-poc/droboticflow-operator-runner:deployed"' in rendered
assert 'name: "droboticflow-registry"' in rendered
assert 'value: "http://droboticflow-worker:3002/worker-api/capacity"' in rendered
def test_poc_gpu_capacity_acceptance_can_expect_available_gpu(tmp_path):
payload = {
"gpu": {
"resourceName": "nvidia.com/gpu",
"available": True,
"schedulableAllocatable": 4,
"maxAllocatable": 4,
"schedulableNodeCount": 1,
"nodes": [{"name": "gpu-node-1", "allocatable": 4, "selectorMatched": True}],
}
}
result, _ = run_capacity_acceptance(
tmp_path,
payload,
{"POC_GPU_CAPACITY_EXPECT_AVAILABLE": "true"},
)
assert result.returncode == 0, result.stderr + result.stdout
assert "gpu.available=true resource=nvidia.com/gpu schedulableAllocatable=4 maxAllocatable=4 nodes=1" in result.stdout
def test_poc_gpu_capacity_acceptance_fails_when_cpu_only_expected_but_gpu_available(tmp_path):
payload = {
"gpu": {
"resourceName": "nvidia.com/gpu",
"available": True,
"schedulableAllocatable": 1,
"maxAllocatable": 1,
"nodes": [{"name": "gpu-node-1", "allocatable": 1, "selectorMatched": True}],
}
}
result, _ = run_capacity_acceptance(tmp_path, payload, {"POC_GPU_CAPACITY_EXPECT_AVAILABLE": "false"})
assert result.returncode == 1
assert "expected CPU-only POC GPU capacity" in result.stderr
def test_poc_gpu_capacity_acceptance_rejects_invalid_inputs_before_kubectl(tmp_path):
result, _ = run_capacity_acceptance(
tmp_path,
cpu_only_capacity_payload(),
{"POC_GPU_CAPACITY_NAME": "bad_name"},
)
assert result.returncode == 1
assert "POC_GPU_CAPACITY_NAME must be a lowercase DNS label" in result.stderr
assert "unexpected kubectl call" not in result.stderr