787 lines
35 KiB
Python
Executable File
787 lines
35 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
|
|
DEFAULT_NAMESPACE = "droboticflow"
|
|
DEFAULT_WORKSPACE_MOUNT_PATH = "/workspace"
|
|
DEFAULT_FIXTURE_MANIFEST_RELATIVE_PATH = "datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"
|
|
DEFAULT_GPU_CAPACITY_WORKER_URL = "http://droboticflow-worker:3002/worker-api/capacity"
|
|
DEFAULT_GPU_CAPACITY_SERVICE_NAME = "droboticflow-worker"
|
|
DEFAULT_GPU_CAPACITY_SERVICE_PORT = 3002
|
|
DEFAULT_API_PUBLIC_SERVICE_NAME = "droboticflow-api-public"
|
|
DEFAULT_AUTH_SECRET_NAME = "droboticflow-auth-secret"
|
|
DEFAULT_GPU_WORKFLOW_EXPECTED_LOG = "gpu request cannot be scheduled"
|
|
MANIFEST_FIXTURE_PATHS = {
|
|
"xplatform": ("dir", (("fixtures", "xplatform"), ("fixtures", "xplatformSourcePath"), ("xplatform", "sourcePath"))),
|
|
"csv": ("file", (("fixtures", "csv"), ("fixtures", "csvInputPath"))),
|
|
"json": ("file", (("fixtures", "json"), ("fixtures", "jsonInputPath"))),
|
|
"lerobot": ("dir", (("fixtures", "lerobot"), ("fixtures", "lerobotSourcePath"))),
|
|
"rosbag": ("any", (("fixtures", "rosbag"), ("fixtures", "rosbagSourcePath"))),
|
|
"model": ("file", (("fixtures", "model"), ("fixtures", "modelJsonPath"))),
|
|
}
|
|
MANIFEST_SHELL_COMMAND_PATHS = (("shellCommand",), ("shell", "command"))
|
|
|
|
|
|
@dataclass
|
|
class Check:
|
|
name: str
|
|
status: str
|
|
message: str
|
|
evidence: dict[str, Any]
|
|
required: bool = True
|
|
|
|
|
|
def env_bool(name: str, default: bool = False) -> bool:
|
|
raw = os.getenv(name, "").strip().lower()
|
|
if not raw:
|
|
return default
|
|
if raw in {"1", "true", "yes", "y", "on"}:
|
|
return True
|
|
if raw in {"0", "false", "no", "n", "off"}:
|
|
return False
|
|
raise SystemExit(f"{name} must be a boolean value")
|
|
|
|
|
|
def run_kubectl(args: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["kubectl", *args],
|
|
input=input_text,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def kubectl_json(args: list[str]) -> dict[str, Any] | None:
|
|
result = run_kubectl([*args, "-o", "json"])
|
|
if result.returncode != 0:
|
|
return None
|
|
try:
|
|
payload = json.loads(result.stdout)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
return payload if isinstance(payload, dict) else None
|
|
|
|
|
|
def decode_secret_value(secret: dict[str, Any], *keys: str) -> str:
|
|
data = secret.get("data")
|
|
if not isinstance(data, dict):
|
|
return ""
|
|
for key in keys:
|
|
raw = data.get(key)
|
|
if not isinstance(raw, str) or not raw:
|
|
continue
|
|
try:
|
|
value = base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip()
|
|
except Exception:
|
|
continue
|
|
if value:
|
|
return value
|
|
return ""
|
|
|
|
|
|
def deployment_check(namespace: str, name: str, expected_tag: str) -> Check:
|
|
payload = kubectl_json(["-n", namespace, "get", "deploy", name])
|
|
if payload is None:
|
|
return Check("deployment:" + name, "failed", f"deployment {namespace}/{name} is missing or unreadable", {})
|
|
|
|
status = payload.get("status") if isinstance(payload.get("status"), dict) else {}
|
|
spec = payload.get("spec") if isinstance(payload.get("spec"), dict) else {}
|
|
template = spec.get("template") if isinstance(spec.get("template"), dict) else {}
|
|
pod_spec = template.get("spec") if isinstance(template.get("spec"), dict) else {}
|
|
containers = pod_spec.get("containers") if isinstance(pod_spec.get("containers"), list) else []
|
|
images = [item.get("image", "") for item in containers if isinstance(item, dict)]
|
|
ready = int(status.get("readyReplicas") or 0)
|
|
replicas = int(status.get("replicas") or spec.get("replicas") or 0)
|
|
|
|
issues: list[str] = []
|
|
if replicas <= 0 or ready < replicas:
|
|
issues.append(f"ready replicas {ready}/{replicas}")
|
|
if expected_tag and not all(str(image).endswith(":" + expected_tag) for image in images):
|
|
issues.append(f"image tag does not match {expected_tag}")
|
|
if issues:
|
|
return Check("deployment:" + name, "failed", "; ".join(issues), {"images": images, "readyReplicas": ready, "replicas": replicas})
|
|
return Check("deployment:" + name, "passed", "deployment is ready", {"images": images, "readyReplicas": ready, "replicas": replicas})
|
|
|
|
|
|
def runner_image_check(namespace: str, expected_tag: str) -> Check:
|
|
payload = kubectl_json(["-n", namespace, "get", "configmap", "droboticflow-config"])
|
|
if payload is None:
|
|
return Check("operator-runner-image", "failed", "configmap droboticflow-config is missing or unreadable", {})
|
|
data = payload.get("data") if isinstance(payload.get("data"), dict) else {}
|
|
image = str(data.get("OPERATOR_RUNNER_IMAGE") or "")
|
|
if not image:
|
|
return Check("operator-runner-image", "failed", "OPERATOR_RUNNER_IMAGE is not configured", {})
|
|
if expected_tag and not image.endswith(":" + expected_tag):
|
|
return Check("operator-runner-image", "failed", f"operator runner image tag does not match {expected_tag}", {"image": image})
|
|
return Check("operator-runner-image", "passed", "operator runner image is configured", {"image": image})
|
|
|
|
|
|
def configmap_data(namespace: str, name: str) -> dict[str, str]:
|
|
payload = kubectl_json(["-n", namespace, "get", "configmap", name])
|
|
if payload is None:
|
|
return {}
|
|
data = payload.get("data")
|
|
return {str(key): str(value) for key, value in data.items()} if isinstance(data, dict) else {}
|
|
|
|
|
|
def positive_int_env(name: str, default: int) -> int:
|
|
raw = os.getenv(name, "").strip()
|
|
if not raw:
|
|
return default
|
|
if not raw.isdigit() or int(raw) <= 0:
|
|
raise SystemExit(f"{name} must be a positive integer")
|
|
return int(raw)
|
|
|
|
|
|
def normalize_http_url(raw: str) -> str:
|
|
value = raw.strip().rstrip("/")
|
|
if not value:
|
|
return ""
|
|
parsed = urllib.parse.urlsplit(value)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or any(character.isspace() for character in value):
|
|
raise SystemExit(f"{raw} must be an absolute http(s) URL without whitespace")
|
|
return value
|
|
|
|
|
|
def is_private_load_balancer_host(host: str) -> bool:
|
|
try:
|
|
return ipaddress.ip_address(host).is_private
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def platform_readiness_base_url(namespace: str) -> str:
|
|
for name in ("POC_PLATFORM_READINESS_API_BASE_URL", "API_BASE_URL", "BASE_URL"):
|
|
value = normalize_http_url(os.getenv(name, ""))
|
|
if value:
|
|
return value
|
|
|
|
service_name = (
|
|
os.getenv("POC_PLATFORM_READINESS_API_PUBLIC_SERVICE_NAME", "").strip()
|
|
or os.getenv("API_PUBLIC_SERVICE_NAME", "").strip()
|
|
or DEFAULT_API_PUBLIC_SERVICE_NAME
|
|
)
|
|
if not service_name or any(character.isspace() for character in service_name):
|
|
raise SystemExit("POC_PLATFORM_READINESS_API_PUBLIC_SERVICE_NAME must be non-empty and contain no whitespace")
|
|
payload = kubectl_json(["-n", namespace, "get", "svc", service_name])
|
|
if payload is None:
|
|
return ""
|
|
status = payload.get("status") if isinstance(payload.get("status"), dict) else {}
|
|
load_balancer = status.get("loadBalancer") if isinstance(status.get("loadBalancer"), dict) else {}
|
|
ingress = load_balancer.get("ingress") if isinstance(load_balancer.get("ingress"), list) else []
|
|
hosts = [str(item.get("ip") or item.get("hostname") or "").strip() for item in ingress if isinstance(item, dict)]
|
|
hosts = [host for host in hosts if host]
|
|
if not hosts:
|
|
return ""
|
|
for host in hosts:
|
|
if not is_private_load_balancer_host(host):
|
|
return f"http://{host}"
|
|
return f"http://{hosts[0]}"
|
|
|
|
|
|
def platform_readiness_token(namespace: str) -> tuple[str, dict[str, Any]]:
|
|
token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
|
|
if token:
|
|
return token, {"tokenSource": "env:DROBOTICFLOW_API_TOKEN"}
|
|
|
|
secret_name = os.getenv("POC_PLATFORM_READINESS_API_TOKEN_SECRET_NAME", DEFAULT_AUTH_SECRET_NAME).strip()
|
|
if not secret_name or any(character.isspace() for character in secret_name):
|
|
raise SystemExit("POC_PLATFORM_READINESS_API_TOKEN_SECRET_NAME must be non-empty and contain no whitespace")
|
|
secret = kubectl_json(["-n", namespace, "get", "secret", secret_name])
|
|
if secret is None:
|
|
return "", {"tokenSource": "secret", "secretName": secret_name}
|
|
token = decode_secret_value(secret, "DROBOTICFLOW_API_TOKEN", "API_TOKEN", "token")
|
|
return token, {"tokenSource": "secret", "secretName": secret_name}
|
|
|
|
|
|
def http_get_json(url: str, token: str, timeout_seconds: int) -> tuple[dict[str, Any] | None, str]:
|
|
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"})
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
try:
|
|
with opener.open(request, timeout=timeout_seconds) as response:
|
|
body = response.read().decode("utf-8")
|
|
except urllib.error.HTTPError as error:
|
|
return None, f"platform readiness API returned HTTP {error.code}"
|
|
except (OSError, urllib.error.URLError, TimeoutError) as error:
|
|
return None, f"platform readiness API request failed: {error}"
|
|
try:
|
|
payload = json.loads(body)
|
|
except json.JSONDecodeError as error:
|
|
return None, f"platform readiness API did not return JSON: {error.msg}"
|
|
return (payload, "") if isinstance(payload, dict) else (None, "platform readiness API payload is not a JSON object")
|
|
|
|
|
|
def gpu_capacity_pod_name() -> str:
|
|
raw = os.getenv("POC_GPU_CAPACITY_NAME", "").strip()
|
|
if raw:
|
|
return raw
|
|
return f"droboticflow-gpu-capacity-readiness-{int(time.time())}"
|
|
|
|
|
|
def gpu_capacity_probe_mode() -> str:
|
|
value = os.getenv("POC_GPU_CAPACITY_PROBE_MODE", "port-forward").strip().lower()
|
|
if value not in {"port-forward", "pod"}:
|
|
raise SystemExit("POC_GPU_CAPACITY_PROBE_MODE must be port-forward or pod")
|
|
return value
|
|
|
|
|
|
def endpoint_path_from_worker_url(raw: str) -> str:
|
|
value = raw.strip()
|
|
if not value:
|
|
return "/worker-api/capacity"
|
|
if value.startswith("/"):
|
|
return value
|
|
parsed = urllib.parse.urlsplit(value)
|
|
path = parsed.path or "/worker-api/capacity"
|
|
return path + (("?" + parsed.query) if parsed.query else "")
|
|
|
|
|
|
def free_local_port() -> int:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
return int(sock.getsockname()[1])
|
|
|
|
|
|
def parse_capacity_payload(raw: str) -> tuple[dict[str, Any] | None, str]:
|
|
last_error = ""
|
|
for line in reversed(raw.splitlines()):
|
|
candidate = line.strip()
|
|
if not candidate.startswith("{"):
|
|
continue
|
|
try:
|
|
payload = json.loads(candidate)
|
|
except json.JSONDecodeError as error:
|
|
last_error = str(error)
|
|
continue
|
|
return (payload, "") if isinstance(payload, dict) else (None, "capacity payload is not a JSON object")
|
|
return None, f"capacity logs did not contain a JSON object{': ' + last_error if last_error else ''}"
|
|
|
|
|
|
def summarize_gpu_capacity(payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
gpu = payload.get("gpu")
|
|
if not isinstance(gpu, dict):
|
|
return {}, "capacity payload is missing gpu object"
|
|
nodes = gpu.get("nodes")
|
|
if not isinstance(nodes, list):
|
|
return {}, "gpu.nodes must be an array"
|
|
|
|
def int_value(name: str) -> int:
|
|
try:
|
|
return int(gpu.get(name) or 0)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"gpu.{name} must be an integer-compatible value")
|
|
|
|
try:
|
|
schedulable_allocatable = int_value("schedulableAllocatable")
|
|
max_allocatable = int_value("maxAllocatable")
|
|
schedulable_node_count = int_value("schedulableNodeCount")
|
|
except ValueError as error:
|
|
return {}, str(error)
|
|
|
|
normalized_nodes: list[dict[str, Any]] = []
|
|
for item in nodes:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
try:
|
|
allocatable = int(item.get("allocatable") or 0)
|
|
except (TypeError, ValueError):
|
|
return {}, "gpu.nodes[].allocatable must be an integer-compatible value"
|
|
normalized_nodes.append(
|
|
{
|
|
"name": str(item.get("name") or ""),
|
|
"allocatable": allocatable,
|
|
"selectorMatched": bool(item.get("selectorMatched")),
|
|
}
|
|
)
|
|
if not normalized_nodes:
|
|
return {}, "gpu.nodes did not contain any node objects"
|
|
|
|
evidence = {
|
|
"resourceName": str(gpu.get("resourceName") or ""),
|
|
"available": bool(gpu.get("available")),
|
|
"schedulableAllocatable": schedulable_allocatable,
|
|
"maxAllocatable": max_allocatable,
|
|
"schedulableNodeCount": schedulable_node_count,
|
|
"nodeCount": len(nodes),
|
|
"nodes": normalized_nodes,
|
|
}
|
|
if not evidence["resourceName"]:
|
|
return evidence, "gpu.resourceName is required"
|
|
if evidence["nodeCount"] <= 0:
|
|
return evidence, "gpu.nodes is empty; capacity endpoint did not report checked nodes"
|
|
return evidence, ""
|
|
|
|
|
|
def gpu_capacity_expectation_check(evidence: dict[str, Any], expected_available: bool) -> str:
|
|
available = bool(evidence.get("available"))
|
|
schedulable = int(evidence.get("schedulableAllocatable") or 0)
|
|
max_allocatable = int(evidence.get("maxAllocatable") or 0)
|
|
if expected_available:
|
|
if available and schedulable > 0 and max_allocatable > 0:
|
|
return ""
|
|
return (
|
|
"expected POC GPU capacity to be available, "
|
|
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}"
|
|
)
|
|
if not available and schedulable == 0 and max_allocatable == 0:
|
|
return ""
|
|
return (
|
|
"expected CPU-only POC GPU capacity, "
|
|
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}"
|
|
)
|
|
|
|
|
|
def gpu_capacity_result_from_logs(raw: str, evidence_base: dict[str, Any], expected_available: bool) -> Check:
|
|
payload, parse_error = parse_capacity_payload(raw)
|
|
if payload is None:
|
|
return Check("gpu-capacity", "failed", parse_error, evidence_base)
|
|
capacity_evidence, payload_error = summarize_gpu_capacity(payload)
|
|
evidence = {**evidence_base, **capacity_evidence}
|
|
if payload_error:
|
|
return Check("gpu-capacity", "failed", payload_error, evidence)
|
|
expectation_error = gpu_capacity_expectation_check(evidence, expected_available)
|
|
if expectation_error:
|
|
return Check("gpu-capacity", "failed", expectation_error, evidence)
|
|
message = "GPU capacity is available as expected" if expected_available else "POC GPU capacity is CPU-only as expected"
|
|
return Check("gpu-capacity", "passed", message, evidence)
|
|
|
|
|
|
def gpu_capacity_pod_check(
|
|
namespace: str,
|
|
data: dict[str, str],
|
|
worker_url: str,
|
|
expected_available: bool,
|
|
timeout_seconds: int,
|
|
http_timeout_seconds: int,
|
|
) -> Check:
|
|
image = os.getenv("POC_GPU_CAPACITY_IMAGE", "").strip() or os.getenv("OPERATOR_RUNNER_IMAGE", "").strip() or data.get("OPERATOR_RUNNER_IMAGE", "")
|
|
if not image:
|
|
return Check("gpu-capacity", "failed", "OPERATOR_RUNNER_IMAGE is not configured for GPU capacity readiness", {})
|
|
if any(character.isspace() for character in image):
|
|
return Check("gpu-capacity", "failed", "POC_GPU_CAPACITY_IMAGE must not contain whitespace", {"image": image})
|
|
image_pull_secret = os.getenv("K8S_IMAGE_PULL_SECRET_NAME", "").strip() or data.get("K8S_IMAGE_PULL_SECRET_NAME", "")
|
|
pod_name = gpu_capacity_pod_name()
|
|
manifest: dict[str, Any] = {
|
|
"apiVersion": "v1",
|
|
"kind": "Pod",
|
|
"metadata": {
|
|
"name": pod_name,
|
|
"namespace": namespace,
|
|
"labels": {"app": "droboticflow", "component": "poc-gpu-capacity-readiness"},
|
|
},
|
|
"spec": {
|
|
"restartPolicy": "Never",
|
|
"containers": [
|
|
{
|
|
"name": "capacity",
|
|
"image": image,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"command": [
|
|
"python",
|
|
"-c",
|
|
"\n".join(
|
|
[
|
|
"import json",
|
|
"import os",
|
|
"import urllib.request",
|
|
'url = os.environ["WORKER_CAPACITY_URL"]',
|
|
'timeout = int(os.environ.get("WORKER_CAPACITY_TIMEOUT_SECONDS", "30"))',
|
|
"with urllib.request.urlopen(url, timeout=timeout) as response:",
|
|
' body = response.read().decode("utf-8")',
|
|
"json.loads(body)",
|
|
"print(body)",
|
|
]
|
|
),
|
|
],
|
|
"env": [
|
|
{"name": "WORKER_CAPACITY_URL", "value": worker_url},
|
|
{"name": "WORKER_CAPACITY_TIMEOUT_SECONDS", "value": str(http_timeout_seconds)},
|
|
],
|
|
}
|
|
],
|
|
},
|
|
}
|
|
if image_pull_secret:
|
|
manifest["spec"]["imagePullSecrets"] = [{"name": image_pull_secret}]
|
|
|
|
evidence_base = {"probeMode": "pod", "podName": pod_name, "image": image, "workerUrl": worker_url, "expectedAvailable": expected_available}
|
|
applied = run_kubectl(["apply", "-f", "-"], input_text=json.dumps(manifest))
|
|
if applied.returncode != 0:
|
|
return Check("gpu-capacity", "failed", "failed to create GPU capacity readiness Pod", {**evidence_base, "stderr": applied.stderr.strip()})
|
|
|
|
try:
|
|
deadline = time.time() + timeout_seconds
|
|
phase = ""
|
|
while time.time() < deadline:
|
|
result = run_kubectl(["-n", namespace, "get", "pod", pod_name, "-o", "jsonpath={.status.phase}"])
|
|
phase = result.stdout.strip() if result.returncode == 0 else ""
|
|
if phase in {"Succeeded", "Failed"}:
|
|
break
|
|
time.sleep(2)
|
|
if phase not in {"Succeeded", "Failed"}:
|
|
return Check("gpu-capacity", "failed", "timed out waiting for GPU capacity readiness Pod", {**evidence_base, "phase": phase or "unknown"})
|
|
|
|
logs_result = run_kubectl(["-n", namespace, "logs", pod_name])
|
|
logs = logs_result.stdout if logs_result.returncode == 0 else logs_result.stderr
|
|
if phase != "Succeeded":
|
|
return Check("gpu-capacity", "failed", "GPU capacity readiness Pod failed", {**evidence_base, "phase": phase, "logs": logs.strip()[-400:]})
|
|
|
|
return gpu_capacity_result_from_logs(logs, evidence_base, expected_available)
|
|
finally:
|
|
run_kubectl(["-n", namespace, "delete", "pod", pod_name, "--ignore-not-found"])
|
|
|
|
|
|
def gpu_capacity_port_forward_check(
|
|
namespace: str,
|
|
worker_url: str,
|
|
expected_available: bool,
|
|
timeout_seconds: int,
|
|
http_timeout_seconds: int,
|
|
) -> Check:
|
|
service_name = os.getenv("POC_GPU_CAPACITY_SERVICE_NAME", DEFAULT_GPU_CAPACITY_SERVICE_NAME).strip()
|
|
service_port = positive_int_env("POC_GPU_CAPACITY_SERVICE_PORT", DEFAULT_GPU_CAPACITY_SERVICE_PORT)
|
|
if not service_name or any(character.isspace() for character in service_name):
|
|
return Check("gpu-capacity", "failed", "POC_GPU_CAPACITY_SERVICE_NAME must be non-empty and contain no whitespace", {})
|
|
local_port = free_local_port()
|
|
endpoint_path = endpoint_path_from_worker_url(worker_url)
|
|
local_url = f"http://127.0.0.1:{local_port}{endpoint_path}"
|
|
evidence_base = {
|
|
"probeMode": "port-forward",
|
|
"serviceName": service_name,
|
|
"servicePort": service_port,
|
|
"localPort": local_port,
|
|
"workerUrl": local_url,
|
|
"expectedAvailable": expected_available,
|
|
}
|
|
process = subprocess.Popen(
|
|
["kubectl", "-n", namespace, "port-forward", f"service/{service_name}", f"{local_port}:{service_port}"],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
try:
|
|
deadline = time.time() + timeout_seconds
|
|
last_error = ""
|
|
while time.time() < deadline:
|
|
if process.poll() is not None:
|
|
stdout, stderr = process.communicate()
|
|
return Check(
|
|
"gpu-capacity",
|
|
"failed",
|
|
"kubectl port-forward for GPU capacity exited early",
|
|
{**evidence_base, "stdout": stdout.strip()[-400:], "stderr": stderr.strip()[-400:]},
|
|
)
|
|
try:
|
|
with opener.open(local_url, timeout=http_timeout_seconds) as response:
|
|
logs = response.read().decode("utf-8")
|
|
return gpu_capacity_result_from_logs(logs, evidence_base, expected_available)
|
|
except (OSError, urllib.error.URLError, TimeoutError) as error:
|
|
last_error = str(error)
|
|
time.sleep(0.25)
|
|
return Check("gpu-capacity", "failed", "timed out waiting for GPU capacity port-forward", {**evidence_base, "lastError": last_error})
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=3)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait(timeout=3)
|
|
|
|
|
|
def gpu_capacity_check(namespace: str) -> Check:
|
|
expected_available = env_bool("POC_GPU_CAPACITY_EXPECT_AVAILABLE", True)
|
|
probe_mode = gpu_capacity_probe_mode()
|
|
timeout_seconds = positive_int_env("POC_GPU_CAPACITY_TIMEOUT_SECONDS", 120)
|
|
http_timeout_seconds = positive_int_env("POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS", 30)
|
|
worker_url = os.getenv("POC_GPU_CAPACITY_WORKER_URL", DEFAULT_GPU_CAPACITY_WORKER_URL).strip()
|
|
if not worker_url or any(character.isspace() for character in worker_url):
|
|
return Check("gpu-capacity", "failed", "POC_GPU_CAPACITY_WORKER_URL must be a non-empty URL without whitespace", {})
|
|
|
|
data = configmap_data(namespace, "droboticflow-config")
|
|
if probe_mode == "pod":
|
|
return gpu_capacity_pod_check(namespace, data, worker_url, expected_available, timeout_seconds, http_timeout_seconds)
|
|
return gpu_capacity_port_forward_check(namespace, worker_url, expected_available, timeout_seconds, http_timeout_seconds)
|
|
|
|
|
|
def platform_gpu_workflow_preflight_check(namespace: str, expected_gpu_available: bool) -> Check:
|
|
enabled = env_bool("POC_PLATFORM_READINESS_ENABLED", True)
|
|
if not enabled:
|
|
return Check(
|
|
"platform-gpu-workflow-preflight",
|
|
"skipped",
|
|
"platform GPU workflow preflight readiness is disabled",
|
|
{"enabled": False},
|
|
required=False,
|
|
)
|
|
|
|
base_url = platform_readiness_base_url(namespace)
|
|
if not base_url:
|
|
return Check(
|
|
"platform-gpu-workflow-preflight",
|
|
"failed",
|
|
"platform readiness API base URL is unavailable",
|
|
{
|
|
"serviceName": os.getenv("POC_PLATFORM_READINESS_API_PUBLIC_SERVICE_NAME", "")
|
|
or os.getenv("API_PUBLIC_SERVICE_NAME", "")
|
|
or DEFAULT_API_PUBLIC_SERVICE_NAME
|
|
},
|
|
)
|
|
token, token_evidence = platform_readiness_token(namespace)
|
|
if not token:
|
|
return Check("platform-gpu-workflow-preflight", "failed", "DROBOTICFLOW_API_TOKEN is unavailable for platform readiness API", token_evidence)
|
|
|
|
timeout_seconds = positive_int_env("POC_PLATFORM_READINESS_HTTP_TIMEOUT_SECONDS", 20)
|
|
payload, error = http_get_json(f"{base_url}/api/acceptance/poc-readiness", token, timeout_seconds)
|
|
evidence: dict[str, Any] = {**token_evidence, "apiBaseUrl": base_url, "expectedGpuAvailable": expected_gpu_available}
|
|
if error:
|
|
return Check("platform-gpu-workflow-preflight", "failed", error, evidence)
|
|
assert payload is not None
|
|
|
|
gpu_gate = payload.get("gpu") if isinstance(payload.get("gpu"), dict) else {}
|
|
workflow_gate = payload.get("gpuWorkflowPreflight") if isinstance(payload.get("gpuWorkflowPreflight"), dict) else {}
|
|
workflow_evidence = workflow_gate.get("evidence") if isinstance(workflow_gate.get("evidence"), dict) else {}
|
|
workflow_run_status = workflow_evidence.get("runStatus") or workflow_evidence.get("status")
|
|
evidence.update(
|
|
{
|
|
"platformStatus": payload.get("status"),
|
|
"gpuStatus": gpu_gate.get("status"),
|
|
"gpuWorkflowPreflightStatus": workflow_gate.get("status"),
|
|
"runId": workflow_evidence.get("runId"),
|
|
"runStatus": workflow_run_status,
|
|
"expectedLogSubstring": workflow_evidence.get("expectedLogSubstring"),
|
|
}
|
|
)
|
|
|
|
if expected_gpu_available:
|
|
if gpu_gate.get("status") != "passed":
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU hardware gate is not passed while GPU capacity is expected", evidence)
|
|
if workflow_gate.get("status") != "passed":
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow gate is not passed while GPU capacity is expected", evidence)
|
|
if not workflow_evidence.get("runId"):
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow gate is missing runId evidence", evidence)
|
|
if workflow_run_status != "completed":
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow gate must record a completed run while GPU capacity is expected", evidence)
|
|
return Check("platform-gpu-workflow-preflight", "passed", "platform GPU workflow completion evidence is ready", evidence)
|
|
|
|
if workflow_gate.get("status") != "passed":
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight gate is not passed", evidence)
|
|
if not workflow_evidence.get("runId"):
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight is missing runId evidence", evidence)
|
|
if workflow_run_status != "failed":
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight must record a failed run while POC is CPU-only", evidence)
|
|
expected_log = os.getenv("POC_GPU_WORKFLOW_EXPECT_LOG_SUBSTRING", DEFAULT_GPU_WORKFLOW_EXPECTED_LOG).strip()
|
|
recorded_log = str(workflow_evidence.get("expectedLogSubstring") or "")
|
|
if expected_log and expected_log not in recorded_log:
|
|
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight log evidence does not match the expected blocker", evidence)
|
|
return Check("platform-gpu-workflow-preflight", "passed", "platform GPU workflow preflight evidence is ready", evidence)
|
|
|
|
|
|
def api_pod_name(namespace: str) -> str:
|
|
result = run_kubectl(["-n", namespace, "get", "pod", "-l", "app=droboticflow-api", "-o", "jsonpath={.items[0].metadata.name}"])
|
|
return result.stdout.strip() if result.returncode == 0 else ""
|
|
|
|
|
|
def normalize_workspace_mount_path(raw: str) -> str:
|
|
value = raw.strip().rstrip("/")
|
|
return value or DEFAULT_WORKSPACE_MOUNT_PATH
|
|
|
|
|
|
def default_fixture_manifest_path(workspace_mount_path: str) -> str:
|
|
return f"{workspace_mount_path}/{DEFAULT_FIXTURE_MANIFEST_RELATIVE_PATH}"
|
|
|
|
|
|
def is_under_workspace(path: str, workspace_mount_path: str) -> bool:
|
|
return path == workspace_mount_path or path.startswith(workspace_mount_path + "/")
|
|
|
|
|
|
def workspace_path_check(namespace: str, workspace_mount_path: str, path: str, kind: str, label: str) -> Check:
|
|
if not is_under_workspace(path, workspace_mount_path):
|
|
return Check(label, "failed", f"{label} must point under WORKSPACE_MOUNT_PATH {workspace_mount_path}", {"path": path, "workspaceMountPath": workspace_mount_path})
|
|
pod = api_pod_name(namespace)
|
|
if not pod:
|
|
return Check(label, "failed", "no droboticflow-api pod is available for workspace validation", {"path": path})
|
|
test_script = {
|
|
"file": 'test -f "$1" && test -r "$1" && test -s "$1"',
|
|
"dir": 'test -d "$1" && test -r "$1"',
|
|
"any": 'test -e "$1" && test -r "$1"',
|
|
}[kind]
|
|
result = run_kubectl(["-n", namespace, "exec", pod, "--", "sh", "-c", test_script, "--", path])
|
|
if result.returncode != 0:
|
|
detail = "readable non-empty file" if kind == "file" else f"readable {kind}"
|
|
return Check(label, "failed", f"{label} is not a {detail} inside the shared workspace PVC", {"path": path})
|
|
return Check(label, "passed", f"{label} is readable in the shared workspace PVC", {"path": path})
|
|
|
|
|
|
def fixture_manifest_check(namespace: str, workspace_mount_path: str, manifest_path: str) -> Check:
|
|
file_check = workspace_path_check(namespace, workspace_mount_path, manifest_path, "file", "fixture-manifest")
|
|
if file_check.status != "passed":
|
|
return file_check
|
|
|
|
content = read_workspace_text(namespace, workspace_mount_path, manifest_path, "fixture-manifest")
|
|
if content is None:
|
|
return Check("fixture-manifest", "failed", "fixture-manifest could not be read from the shared workspace PVC", {"path": manifest_path})
|
|
try:
|
|
payload = json.loads(content)
|
|
except json.JSONDecodeError as error:
|
|
return Check(
|
|
"fixture-manifest",
|
|
"failed",
|
|
f"fixture-manifest is not valid JSON: {error.msg}",
|
|
{"path": manifest_path, "line": error.lineno, "column": error.colno},
|
|
)
|
|
if not isinstance(payload, dict):
|
|
return Check("fixture-manifest", "failed", "fixture-manifest must be a JSON object", {"path": manifest_path})
|
|
|
|
missing: list[str] = []
|
|
path_errors: list[dict[str, Any]] = []
|
|
fixtures: dict[str, str] = {}
|
|
for name, (kind, paths) in MANIFEST_FIXTURE_PATHS.items():
|
|
value = manifest_config_text(payload, paths)
|
|
if not value:
|
|
missing.append(name)
|
|
continue
|
|
fixtures[name] = value
|
|
check = workspace_path_check(namespace, workspace_mount_path, value, kind, "fixture:" + name)
|
|
if check.status != "passed":
|
|
path_errors.append({"name": name, "path": value, "message": check.message})
|
|
|
|
shell_command = manifest_config_text(payload, MANIFEST_SHELL_COMMAND_PATHS)
|
|
if not shell_command:
|
|
missing.append("shellCommand")
|
|
|
|
if missing or path_errors:
|
|
details: list[str] = []
|
|
if missing:
|
|
details.append("missing fields: " + ", ".join(missing))
|
|
if path_errors:
|
|
details.append("unready paths: " + "; ".join(f"{item['name']}={item['path']}" for item in path_errors))
|
|
return Check(
|
|
"fixture-manifest",
|
|
"failed",
|
|
"fixture-manifest is incomplete: " + "; ".join(details),
|
|
{
|
|
"path": manifest_path,
|
|
"fixtures": fixtures,
|
|
"missing": missing,
|
|
"pathErrors": path_errors,
|
|
"shellCommandConfigured": bool(shell_command),
|
|
},
|
|
)
|
|
return Check(
|
|
"fixture-manifest",
|
|
"passed",
|
|
"fixture-manifest and real fixture paths are ready",
|
|
{"path": manifest_path, "fixtures": fixtures, "shellCommandConfigured": True},
|
|
)
|
|
|
|
|
|
def read_workspace_text(namespace: str, workspace_mount_path: str, path: str, label: str) -> str | None:
|
|
if not is_under_workspace(path, workspace_mount_path):
|
|
return None
|
|
pod = api_pod_name(namespace)
|
|
if not pod:
|
|
return None
|
|
result = run_kubectl(["-n", namespace, "exec", pod, "--", "cat", path])
|
|
if result.returncode != 0:
|
|
return None
|
|
return result.stdout
|
|
|
|
|
|
def manifest_nested_value(payload: dict[str, Any], path: tuple[str, ...]) -> Any:
|
|
current: Any = payload
|
|
for key in path:
|
|
if not isinstance(current, dict) or key not in current:
|
|
return None
|
|
current = current[key]
|
|
return current
|
|
|
|
|
|
def manifest_config_text(payload: dict[str, Any], paths: tuple[tuple[str, ...], ...]) -> str:
|
|
for path in paths:
|
|
value = manifest_nested_value(payload, path)
|
|
if value is None:
|
|
continue
|
|
if isinstance(value, str):
|
|
return value.strip()
|
|
return ""
|
|
return ""
|
|
|
|
|
|
def required_external_enabled(target: str) -> bool:
|
|
raw = os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", "bos,mongo")
|
|
return target in {part.strip().lower() for part in raw.split(",") if part.strip()}
|
|
|
|
|
|
def main() -> None:
|
|
namespace = os.getenv("K8S_NAMESPACE", DEFAULT_NAMESPACE)
|
|
expected_tag = os.getenv("IMAGE_TAG", "").strip()
|
|
workspace_mount_path = normalize_workspace_mount_path(os.getenv("WORKSPACE_MOUNT_PATH", DEFAULT_WORKSPACE_MOUNT_PATH))
|
|
manifest_path = os.getenv("POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH", "").strip() or default_fixture_manifest_path(workspace_mount_path)
|
|
allow_generated = env_bool("POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", False)
|
|
expected_gpu_available = env_bool("POC_GPU_CAPACITY_EXPECT_AVAILABLE", True)
|
|
|
|
checks = [
|
|
deployment_check(namespace, "droboticflow-api", expected_tag),
|
|
deployment_check(namespace, "droboticflow-worker", expected_tag),
|
|
runner_image_check(namespace, expected_tag),
|
|
gpu_capacity_check(namespace),
|
|
platform_gpu_workflow_preflight_check(namespace, expected_gpu_available),
|
|
]
|
|
if allow_generated:
|
|
checks.append(
|
|
Check(
|
|
"fixture-manifest",
|
|
"failed",
|
|
"generated fixtures are enabled for diagnostics; strict POC readiness requires real fixture evidence",
|
|
{"allowGeneratedFixtures": True},
|
|
required=True,
|
|
)
|
|
)
|
|
else:
|
|
checks.append(fixture_manifest_check(namespace, workspace_mount_path, manifest_path))
|
|
|
|
failed_required = [check for check in checks if check.required and check.status == "failed"]
|
|
payload = {
|
|
"status": "ready" if not failed_required else "blocked",
|
|
"namespace": namespace,
|
|
"workspaceMountPath": workspace_mount_path,
|
|
"expectedImageTag": expected_tag,
|
|
"requiredExternals": sorted({part.strip().lower() for part in os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", "bos,mongo").split(",") if part.strip()}),
|
|
"checks": [check.__dict__ for check in checks],
|
|
"missing": [{"name": check.name, "message": check.message, "evidence": check.evidence} for check in failed_required],
|
|
"nextActions": [],
|
|
}
|
|
if any(check.name == "fixture-manifest" and check.status == "failed" for check in checks):
|
|
if allow_generated:
|
|
payload["nextActions"].append(
|
|
"Unset POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES and create a real fixture manifest with npm run acceptance:poc-fixtures using non-generated local or /workspace fixture data."
|
|
)
|
|
else:
|
|
payload["nextActions"].append("Create a real fixture manifest with npm run acceptance:poc-fixtures using non-generated local or /workspace fixture data.")
|
|
if any(check.name == "platform-gpu-workflow-preflight" and check.status == "failed" for check in checks):
|
|
payload["nextActions"].append("Run npm run acceptance:poc-gpu-workflow with a real DROBOTICFLOW_API_TOKEN, then rerun npm run acceptance:poc-readiness.")
|
|
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
|
raise SystemExit(0 if payload["status"] == "ready" else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|