From 9f77602ffeefade6e774f495fc2970837c473656 Mon Sep 17 00:00:00 2001 From: eust-w Date: Fri, 26 Jun 2026 04:07:55 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20Respect=20GPU=20selector=20in=20?= =?UTF-8?q?acceptance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/deployment.md | 2 +- docs/operations.md | 2 +- docs/production-readiness.md | 2 +- scripts/gpu-acceptance-test.sh | 32 ++++++++-- tests/backend/gpu_acceptance_script_test.py | 68 ++++++++++++++++++++- 5 files changed, 95 insertions(+), 11 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 77e1fea7..6a94e4ff 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -237,7 +237,7 @@ http://:30080 - `scripts/object-storage-acceptance.mjs` runs browser create, preview, rename, upload, delete, and optional video playback checks against a live storage connection. It never rewrites storage connection metadata; provide `OBJECT_STORAGE_CONNECTION_ID`, `OBJECT_STORAGE_CONNECTION_NAME`, or `OBJECT_STORAGE_BUCKET` explicitly. `OBJECT_STORAGE_ACCEPTANCE_PAYLOAD_FILE` is required and must point to a non-empty real UTF-8 text or JSON payload used for create/upload/preview checks; set `OBJECT_STORAGE_ACCEPTANCE_CONTENT_TYPE` when auto-detection is not enough. Set `OBJECT_STORAGE_VIDEO_KEY` only when an existing playable object should be validated. `PLAYWRIGHT_IGNORE_HTTPS_ERRORS` must be an explicit boolean spelling when set. The script supports `DROBOTICFLOW_API_TOKEN` for token-protected non-session APIs and `DROBOTICFLOW_ACCEPTANCE_EMAIL` / `DROBOTICFLOW_ACCEPTANCE_PASSWORD` for session-authenticated browser access; session-auth object-storage management checks require an organization admin account. - `scripts/playwright-preview-operators-check.mjs` runs browser/operator preview acceptance with caller-supplied payloads and real source paths. Provide `PREVIEW_CHECK_CSV_PAYLOAD_FILE`, `PREVIEW_CHECK_TEXT_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_EXTRACT_FIELD`, `PREVIEW_CHECK_JSON_FILTER_FIELD`, `PREVIEW_CHECK_JSON_FIELD_MAP_JSON`, `PREVIEW_CHECK_SCAN_SOURCE_PATH`, `PREVIEW_CHECK_ROSBAG_SOURCE_PATH`, and `PREVIEW_CHECK_LEROBOT_SOURCE_PATH`. The script no longer generates built-in CSV/text/JSON/ROSBag/LeRobot data. Optional `PREVIEW_CHECK_SCAN_MIN_FILES` must be an exact safe positive integer. - `scripts/xplatform-live-acceptance.py` runs the X-platform chain only when the caller provides the real source path plus explicit acceptance criteria. Provide `XPLATFORM_RULES_JSON`, all `XPLATFORM_MIN_*` counts, finite normalized `XPLATFORM_MIN_CLARITY`, `XPLATFORM_MAX_SHAKE`, and `XPLATFORM_MAX_HAND_OCCLUSION`, finite positive `XPLATFORM_CLARITY_SCALE` and `XPLATFORM_FRAME_INTERVAL_SECONDS`, and exact safe positive integer values for `XPLATFORM_SEGMENT_DURATION_SECONDS` and every `XPLATFORM_MIN_*` count; the script has no built-in sampling rule or threshold fallback. -- `npm run acceptance:gpu` loads the same deploy environment resolver as the POC deploy scripts, so `dev` resolves to the POC kubeconfig when `DEPLOY_ENV` is unset. It then creates a temporary GPU Job in the target namespace and runs `nvidia-smi`. Override `GPU_ACCEPTANCE_IMAGE`, `GPU_ACCEPTANCE_TIMEOUT`, `GPU_RESOURCE_NAME`, `GPU_RESOURCE_COUNT`, and the `OPERATOR_GPU_*` scheduling variables when the cluster uses custom GPU images, wait windows, or labels. The script validates namespace, image, timeout, Job name, resource name/count, node selector, tolerations JSON, and runtime class before any `kubectl` call so malformed GPU acceptance inputs fail locally instead of mutating the cluster with a bad manifest. It also checks node allocatable resources before creating the Job; if no node reports at least the requested GPU count, the script exits with the checked node/resource evidence and the environment must be treated as GPU-not-accepted. Dynamic manifest strings are rendered as quoted YAML scalars. +- `npm run acceptance:gpu` loads the same deploy environment resolver as the POC deploy scripts, so `dev` resolves to the POC kubeconfig when `DEPLOY_ENV` is unset. It then creates a temporary GPU Job in the target namespace and runs `nvidia-smi`. Override `GPU_ACCEPTANCE_IMAGE`, `GPU_ACCEPTANCE_TIMEOUT`, `GPU_RESOURCE_NAME`, `GPU_RESOURCE_COUNT`, and the `OPERATOR_GPU_*` scheduling variables when the cluster uses custom GPU images, wait windows, or labels. The script validates namespace, image, timeout, Job name, resource name/count, node selector, tolerations JSON, and runtime class before any `kubectl` call so malformed GPU acceptance inputs fail locally instead of mutating the cluster with a bad manifest. It also requires a selector-matched, non-unschedulable node with at least the requested GPU count before creating the Job; if none exists, the script exits with checked node/resource/selector/unschedulable evidence and the environment must be treated as GPU-not-accepted. Dynamic manifest strings are rendered as quoted YAML scalars. - `npm run acceptance:poc-gpu-capacity` starts a temporary in-cluster Pod with the deployed operator-runner image and calls the deployed worker service at `/worker-api/capacity`. It is the no-login POC capacity acceptance check: the default expectation is CPU-only POC, and `POC_GPU_CAPACITY_EXPECT_AVAILABLE=true` should be used only after real GPU hardware is installed. - `npm run acceptance:poc-gpu-workflow` creates a real POC project/workflow/run with the source node requesting `gpu=1` and expects the worker to fail fast with `gpu request cannot be scheduled` while POC has no GPU nodes. Because POC write APIs require a real session, set `DROBOTICFLOW_ACCEPTANCE_EMAIL` and `DROBOTICFLOW_ACCEPTANCE_PASSWORD` before running it. Override `POC_GPU_WORKFLOW_SOURCE_PATH` only when the default real fixture path is not present. - Deployed-environment checks run from a local shell or external CI by invoking the acceptance scripts directly. Configure `WORKFLOW_ACCEPTANCE_SOURCE_PATH` for live workflow acceptance, a kubeconfig for GPU acceptance, and optionally `DROBOTICFLOW_API_TOKEN`, `DROBOTICFLOW_ACCEPTANCE_EMAIL`, `DROBOTICFLOW_ACCEPTANCE_PASSWORD`, `OBJECT_STORAGE_CONNECTION_ID`, `OBJECT_STORAGE_CONNECTION_NAME`, `OBJECT_STORAGE_BUCKET`, `OBJECT_STORAGE_ACCEPTANCE_PAYLOAD_FILE`, `OBJECT_STORAGE_ACCEPTANCE_CONTENT_TYPE`, `OBJECT_STORAGE_VIDEO_KEY`, `PREVIEW_CHECK_CSV_PAYLOAD_FILE`, `PREVIEW_CHECK_TEXT_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_PAYLOAD_FILE`, the preview JSON field settings, the preview scan/ROSBag/LeRobot source paths, and the X-platform source path plus explicit rules/threshold/count inputs for authenticated acceptance tests. diff --git a/docs/operations.md b/docs/operations.md index 9cf14757..2079c45e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -101,7 +101,7 @@ Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Set `OPERATOR_AL ## Acceptance Checks -Run acceptance checks after a deployment from a local shell or external CI. Live workflow acceptance requires `WORKFLOW_ACCEPTANCE_SOURCE_PATH`, a real path visible to operator-runner Jobs. GPU acceptance loads `scripts/load-deploy-env.sh`, so on `dev` it uses the POC kubeconfig when `DEPLOY_ENV` is unset; other environments still require a kubeconfig with permission to create a temporary Job in `K8S_NAMESPACE`. The GPU acceptance wrapper validates namespace, image, timeout, GPU count, selector, tolerations JSON, resource name, Job name, and runtime class before any `kubectl` call; invalid acceptance inputs fail locally instead of creating a malformed Job. It also checks Node allocatable GPU before creating its probe Job, so a CPU-only cluster reports a hardware-capacity blocker explicitly. The deployed worker exposes the same allocatable-node view at `GET /worker-api/capacity`; API callers reach it through the normal worker API proxy and should expect `gpu.available=false` on POC until the cluster has schedulable `nvidia.com/gpu` resources. +Run acceptance checks after a deployment from a local shell or external CI. Live workflow acceptance requires `WORKFLOW_ACCEPTANCE_SOURCE_PATH`, a real path visible to operator-runner Jobs. GPU acceptance loads `scripts/load-deploy-env.sh`, so on `dev` it uses the POC kubeconfig when `DEPLOY_ENV` is unset; other environments still require a kubeconfig with permission to create a temporary Job in `K8S_NAMESPACE`. The GPU acceptance wrapper validates namespace, image, timeout, GPU count, selector, tolerations JSON, resource name, Job name, and runtime class before any `kubectl` call; invalid acceptance inputs fail locally instead of creating a malformed Job. It also checks for a selector-matched, non-unschedulable Node with enough allocatable GPU before creating its probe Job, so a CPU-only cluster or selector mismatch reports a hardware-capacity blocker explicitly. The deployed worker exposes the same allocatable-node view at `GET /worker-api/capacity`; API callers reach it through the normal worker API proxy and should expect `gpu.available=false` on POC until the cluster has schedulable `nvidia.com/gpu` resources. `npm run acceptance:poc-gpu-capacity` verifies the deployed POC worker capacity endpoint from inside the cluster. It creates a short-lived Pod with the deployed `OPERATOR_RUNNER_IMAGE`, calls `http://droboticflow-worker:3002/worker-api/capacity`, and requires node-level capacity evidence. The default expectation is CPU-only POC (`gpu.available=false`, zero schedulable GPU, zero max single-node GPU); set `POC_GPU_CAPACITY_EXPECT_AVAILABLE=true` only after a real GPU node pool and NVIDIA runtime/device plugin are installed. diff --git a/docs/production-readiness.md b/docs/production-readiness.md index d5d03297..f61aef08 100644 --- a/docs/production-readiness.md +++ b/docs/production-readiness.md @@ -53,7 +53,7 @@ This note tracks the production-facing safeguards that must stay true for Drobod - Operator-runner resource samples include CPU seconds, process memory, GPU utilization, GPU memory used/total, GPU count, and GPU model when `nvidia-smi` is available. - The monitoring page displays GPU utilization, GPU memory, and GPU device information. - `GET /worker-api/capacity` exposes the worker's current Kubernetes Node allocatable GPU snapshot, including matched nodes, schedulable allocatable count, max single-node GPU count, and `available`. The Settings page displays this snapshot, workflow node GPU inputs surface a warning when the current cluster has no schedulable GPU, and the workflow run action blocks GPU-requesting workflows when the loaded worker capacity proves they cannot be scheduled. -- `npm run acceptance:gpu` loads the deployment environment resolver first, so `dev` runs against POC by default when `DEPLOY_ENV` is unset. It remains the real acceptance check for allocatable GPU resources, scheduling, runtime, and `nvidia-smi`. It rejects invalid namespace, image, timeout, GPU count, selector, tolerations, resource-name, Job-name, and runtime-class inputs before it calls `kubectl`, verifies that at least one node has the requested allocatable GPU count before creating a Job, and quotes dynamic manifest values before rendering the temporary Job. A cluster with no matching allocatable GPU resources is GPU-not-accepted, not a skipped success. +- `npm run acceptance:gpu` loads the deployment environment resolver first, so `dev` runs against POC by default when `DEPLOY_ENV` is unset. It remains the real acceptance check for allocatable GPU resources, scheduling, runtime, and `nvidia-smi`. It rejects invalid namespace, image, timeout, GPU count, selector, tolerations, resource-name, Job-name, and runtime-class inputs before it calls `kubectl`, verifies that at least one selector-matched and non-unschedulable node has the requested allocatable GPU count before creating a Job, and quotes dynamic manifest values before rendering the temporary Job. A cluster with no matching allocatable GPU resources is GPU-not-accepted, not a skipped success. - `npm run acceptance:poc-gpu-capacity` validates the deployed POC worker capacity endpoint from inside the cluster with the currently configured operator-runner image. The default accepted result is CPU-only capacity (`gpu.available=false`) until POC has a real GPU node pool. - `npm run acceptance:poc-readiness` includes deployed worker GPU capacity evidence through a default `kubectl port-forward` probe, so every POC readiness report states whether the current hardware expectation is CPU-only or GPU-available without creating a Kubernetes probe Pod unless `POC_GPU_CAPACITY_PROBE_MODE=pod` is explicitly set. - `npm run acceptance:poc-gpu-workflow` verifies the worker preflight path through a real POC workflow run. It requires a session-authenticated acceptance user and expects `gpu request cannot be scheduled` while POC remains CPU-only. diff --git a/scripts/gpu-acceptance-test.sh b/scripts/gpu-acceptance-test.sh index f4afe8aa..4f4feaed 100755 --- a/scripts/gpu-acceptance-test.sh +++ b/scripts/gpu-acceptance-test.sh @@ -217,7 +217,7 @@ if ! kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1; then fi nodes_json="$(kubectl get nodes -o json)" -if ! NODES_JSON="$nodes_json" python3 - "$GPU_RESOURCE_NAME" "$GPU_RESOURCE_COUNT" <<'PY' +if ! NODES_JSON="$nodes_json" OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" python3 - "$GPU_RESOURCE_NAME" "$GPU_RESOURCE_COUNT" <<'PY' import json import os import sys @@ -225,20 +225,42 @@ import sys resource = sys.argv[1] requested = int(sys.argv[2]) payload = json.loads(os.environ["NODES_JSON"]) + +selector: dict[str, str] = {} +raw_selector = os.environ.get("OPERATOR_GPU_NODE_SELECTOR", "").strip() +if raw_selector: + for item in raw_selector.split(","): + item = item.strip() + if not item: + continue + key, value = (part.strip() for part in item.split("=", 1)) + selector[key] = value + +def selector_matches(labels: dict[str, str]) -> bool: + for key, value in selector.items(): + if labels.get(key) != value: + return False + return True + checked: list[str] = [] for node in payload.get("items", []): name = node["metadata"]["name"] + labels = node.get("metadata", {}).get("labels", {}) + labels = labels if isinstance(labels, dict) else {} + selector_matched = selector_matches(labels) + unschedulable = bool(node.get("spec", {}).get("unschedulable", False)) allocatable = node.get("status", {}).get("allocatable", {}) value = str(allocatable.get(resource, "0")) try: count = int(value) except ValueError: count = 0 - checked.append(f"{name} {resource}={value}") - if count >= requested: - print(f"{name} {resource}={value}") + checked.append(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}") + if count >= requested and selector_matched and not unschedulable: + print(f"{name} {resource}={value} selectorMatched={str(selector_matched).lower()} unschedulable={str(unschedulable).lower()}") raise SystemExit(0) -print(f"GPU readiness unavailable: no node reports allocatable {resource}>={requested}", file=sys.stderr) +selector_summary = ",".join(f"{key}={selector[key]}" for key in sorted(selector)) if selector else "" +print(f"GPU readiness unavailable: no schedulable node reports allocatable {resource}>={requested} with selector {selector_summary}", file=sys.stderr) if checked: print("Checked nodes:", file=sys.stderr) for item in checked: diff --git a/tests/backend/gpu_acceptance_script_test.py b/tests/backend/gpu_acceptance_script_test.py index 79a851a9..8e8fc1af 100644 --- a/tests/backend/gpu_acceptance_script_test.py +++ b/tests/backend/gpu_acceptance_script_test.py @@ -1,30 +1,48 @@ import os import pathlib import subprocess +import json ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "gpu-acceptance-test.sh" -def write_kubectl_stub(tmp_path: pathlib.Path, capture_manifest: pathlib.Path | None = None) -> pathlib.Path: +def default_nodes_payload() -> dict: + return { + "items": [ + { + "metadata": {"name": "gpu-node-1", "labels": {"accelerator": "nvidia"}}, + "status": {"allocatable": {"nvidia.com/gpu": "2"}}, + } + ] + } + + +def write_kubectl_stub( + tmp_path: pathlib.Path, + capture_manifest: pathlib.Path | None = None, + nodes_payload: dict | None = None, +) -> pathlib.Path: stub_bin = tmp_path / "bin" stub_bin.mkdir() kubectl = stub_bin / "kubectl" capture_line = "capture_manifest = None\n" if capture_manifest is not None: capture_line = f"capture_manifest = {str(capture_manifest)!r}\n" + nodes_payload = nodes_payload or default_nodes_payload() kubectl.write_text( "#!/usr/bin/env python3\n" "import json\n" "import shutil\n" "import sys\n" f"{capture_line}" + f"nodes_payload = {json.dumps(nodes_payload)!r}\n" "args = sys.argv[1:]\n" "if args[:2] == ['get', 'namespace']:\n" " raise SystemExit(0)\n" "if args == ['get', 'nodes', '-o', 'json']:\n" - " print(json.dumps({'items': [{'metadata': {'name': 'gpu-node-1'}, 'status': {'allocatable': {'nvidia.com/gpu': '2'}}}]}))\n" + " print(json.dumps(json.loads(nodes_payload)))\n" " raise SystemExit(0)\n" "if args[:2] == ['apply', '-f']:\n" " if capture_manifest:\n" @@ -54,8 +72,9 @@ def run_gpu_acceptance( extra_env: dict[str, str], capture_manifest: pathlib.Path | None = None, default_name: bool = True, + nodes_payload: dict | None = None, ) -> subprocess.CompletedProcess: - stub_bin = write_kubectl_stub(tmp_path, capture_manifest=capture_manifest) + stub_bin = write_kubectl_stub(tmp_path, capture_manifest=capture_manifest, nodes_payload=nodes_payload) env = os.environ.copy() env.update( { @@ -137,6 +156,49 @@ def test_gpu_acceptance_renders_valid_scheduling_manifest(tmp_path): assert 'runtimeClassName: "nvidia"' in rendered +def test_gpu_acceptance_rejects_selector_mismatch_before_creating_job(tmp_path): + nodes_payload = { + "items": [ + { + "metadata": {"name": "gpu-node-1", "labels": {"accelerator": "amd"}}, + "status": {"allocatable": {"nvidia.com/gpu": "4"}}, + } + ] + } + result = run_gpu_acceptance( + tmp_path, + {"OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia"}, + nodes_payload=nodes_payload, + ) + + assert result.returncode == 1 + assert "selector accelerator=nvidia" in result.stderr + assert "selectorMatched=false" in result.stderr + assert "unexpected kubectl call" not in result.stderr + + +def test_gpu_acceptance_rejects_unschedulable_gpu_node_before_creating_job(tmp_path): + nodes_payload = { + "items": [ + { + "metadata": {"name": "gpu-node-1", "labels": {"accelerator": "nvidia"}}, + "spec": {"unschedulable": True}, + "status": {"allocatable": {"nvidia.com/gpu": "4"}}, + } + ] + } + result = run_gpu_acceptance( + tmp_path, + {"OPERATOR_GPU_NODE_SELECTOR": "accelerator=nvidia"}, + nodes_payload=nodes_payload, + ) + + assert result.returncode == 1 + assert "unschedulable=true" in result.stderr + assert "selectorMatched=true" in result.stderr + assert "unexpected kubectl call" not in result.stderr + + def test_gpu_acceptance_loads_deploy_env_before_defaults(tmp_path): env_file = ROOT / ".env.poc.local" original = env_file.read_text(encoding="utf-8") if env_file.exists() else None