forked from longtao.wu/droboticflow
171 lines
6.2 KiB
Python
171 lines
6.2 KiB
Python
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "poc-gpu-capacity-acceptance.sh"
|
|
|
|
|
|
def write_kubectl_stub(tmp_path: pathlib.Path, capacity_payload: dict, capture_manifest: pathlib.Path) -> pathlib.Path:
|
|
stub_bin = tmp_path / "bin"
|
|
stub_bin.mkdir()
|
|
kubectl = stub_bin / "kubectl"
|
|
kubectl.write_text(
|
|
f"""#!/usr/bin/env python3
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import sys
|
|
|
|
args = sys.argv[1:]
|
|
capture_manifest = pathlib.Path({str(capture_manifest)!r})
|
|
capacity_payload = {json.dumps(capacity_payload)!r}
|
|
|
|
if args == ["get", "namespace", "droboticflow"]:
|
|
raise SystemExit(0)
|
|
if args == ["-n", "droboticflow", "get", "configmap", "droboticflow-config", "-o", "jsonpath={{.data.OPERATOR_RUNNER_IMAGE}}"]:
|
|
print("ccr.example.test/dataflow-poc/droboticflow-operator-runner:deployed", end="")
|
|
raise SystemExit(0)
|
|
if args == ["-n", "droboticflow", "get", "configmap", "droboticflow-config", "-o", "jsonpath={{.data.K8S_IMAGE_PULL_SECRET_NAME}}"]:
|
|
print("droboticflow-registry", end="")
|
|
raise SystemExit(0)
|
|
if args[:2] == ["apply", "-f"]:
|
|
shutil.copyfile(args[2], capture_manifest)
|
|
print("pod/droboticflow-gpu-capacity-test created")
|
|
raise SystemExit(0)
|
|
if args == ["-n", "droboticflow", "get", "pod", "droboticflow-gpu-capacity-test", "-o", "jsonpath={{.status.phase}}"]:
|
|
print("Succeeded", end="")
|
|
raise SystemExit(0)
|
|
if args == ["-n", "droboticflow", "logs", "droboticflow-gpu-capacity-test"]:
|
|
print(json.dumps(json.loads(capacity_payload), separators=(",", ":")))
|
|
raise SystemExit(0)
|
|
if args[:4] == ["-n", "droboticflow", "delete", "pod"]:
|
|
raise SystemExit(0)
|
|
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
|
|
raise SystemExit(97)
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
return stub_bin
|
|
|
|
|
|
def run_capacity_acceptance(
|
|
tmp_path: pathlib.Path,
|
|
capacity_payload: dict,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
|
|
manifest = tmp_path / "capacity-pod.yaml"
|
|
stub_bin = write_kubectl_stub(tmp_path, capacity_payload, manifest)
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"DEPLOY_ENV": "poc",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
|
|
"POC_GPU_CAPACITY_NAME": "droboticflow-gpu-capacity-test",
|
|
"POC_GPU_CAPACITY_TIMEOUT_SECONDS": "1",
|
|
"POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS": "1",
|
|
}
|
|
)
|
|
env.pop("OPERATOR_RUNNER_IMAGE", None)
|
|
env.pop("POC_GPU_CAPACITY_IMAGE", None)
|
|
if extra_env:
|
|
env.update(extra_env)
|
|
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
return result, manifest
|
|
|
|
|
|
def cpu_only_capacity_payload() -> dict:
|
|
return {
|
|
"gpu": {
|
|
"resourceName": "nvidia.com/gpu",
|
|
"available": False,
|
|
"schedulableAllocatable": 0,
|
|
"maxAllocatable": 0,
|
|
"schedulableNodeCount": 0,
|
|
"nodes": [
|
|
{"name": "192.168.80.249", "allocatable": 0, "selectorMatched": True},
|
|
{"name": "192.168.81.47", "allocatable": 0, "selectorMatched": True},
|
|
],
|
|
}
|
|
}
|
|
|
|
|
|
def available_capacity_payload() -> dict:
|
|
return {
|
|
"gpu": {
|
|
"resourceName": "nvidia.com/gpu",
|
|
"available": True,
|
|
"schedulableAllocatable": 4,
|
|
"maxAllocatable": 4,
|
|
"schedulableNodeCount": 1,
|
|
"nodes": [{"name": "gpu-node-1", "allocatable": 4, "selectorMatched": True}],
|
|
}
|
|
}
|
|
|
|
|
|
def test_poc_gpu_capacity_acceptance_reads_deployed_worker_capacity(tmp_path):
|
|
result, manifest = run_capacity_acceptance(tmp_path, available_capacity_payload())
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "gpu.available=true resource=nvidia.com/gpu schedulableAllocatable=4 maxAllocatable=4 nodes=1" in result.stdout
|
|
assert "node gpu-node-1 nvidia.com/gpu=4 selectorMatched=true" in result.stdout
|
|
rendered = manifest.read_text(encoding="utf-8")
|
|
assert 'name: "droboticflow-gpu-capacity-test"' in rendered
|
|
assert 'namespace: "droboticflow"' in rendered
|
|
assert 'image: "ccr.example.test/dataflow-poc/droboticflow-operator-runner:deployed"' in rendered
|
|
assert 'name: "droboticflow-registry"' in rendered
|
|
assert 'value: "http://droboticflow-worker:3002/worker-api/capacity"' in rendered
|
|
|
|
|
|
def test_poc_gpu_capacity_acceptance_can_expect_available_gpu(tmp_path):
|
|
payload = {
|
|
"gpu": {
|
|
"resourceName": "nvidia.com/gpu",
|
|
"available": True,
|
|
"schedulableAllocatable": 4,
|
|
"maxAllocatable": 4,
|
|
"schedulableNodeCount": 1,
|
|
"nodes": [{"name": "gpu-node-1", "allocatable": 4, "selectorMatched": True}],
|
|
}
|
|
}
|
|
result, _ = run_capacity_acceptance(
|
|
tmp_path,
|
|
payload,
|
|
{"POC_GPU_CAPACITY_EXPECT_AVAILABLE": "true"},
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "gpu.available=true resource=nvidia.com/gpu schedulableAllocatable=4 maxAllocatable=4 nodes=1" in result.stdout
|
|
|
|
|
|
def test_poc_gpu_capacity_acceptance_fails_when_cpu_only_expected_but_gpu_available(tmp_path):
|
|
payload = {
|
|
"gpu": {
|
|
"resourceName": "nvidia.com/gpu",
|
|
"available": True,
|
|
"schedulableAllocatable": 1,
|
|
"maxAllocatable": 1,
|
|
"nodes": [{"name": "gpu-node-1", "allocatable": 1, "selectorMatched": True}],
|
|
}
|
|
}
|
|
result, _ = run_capacity_acceptance(tmp_path, payload, {"POC_GPU_CAPACITY_EXPECT_AVAILABLE": "false"})
|
|
|
|
assert result.returncode == 1
|
|
assert "expected CPU-only POC GPU capacity" in result.stderr
|
|
|
|
|
|
def test_poc_gpu_capacity_acceptance_rejects_invalid_inputs_before_kubectl(tmp_path):
|
|
result, _ = run_capacity_acceptance(
|
|
tmp_path,
|
|
cpu_only_capacity_payload(),
|
|
{"POC_GPU_CAPACITY_NAME": "bad_name"},
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "POC_GPU_CAPACITY_NAME must be a lowercase DNS label" in result.stderr
|
|
assert "unexpected kubectl call" not in result.stderr
|