droboticflow/scripts/poc_acceptance_readiness.py
2026-07-07 18:57:49 +08:00

787 lines
35 KiB
Python
Executable File

#!/usr/bin/env python3
from __future__ import annotations
import base64
import ipaddress
import json
import os
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from typing import Any
DEFAULT_NAMESPACE = "droboticflow"
DEFAULT_WORKSPACE_MOUNT_PATH = "/workspace"
DEFAULT_FIXTURE_MANIFEST_RELATIVE_PATH = "datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"
DEFAULT_GPU_CAPACITY_WORKER_URL = "http://droboticflow-worker:3002/worker-api/capacity"
DEFAULT_GPU_CAPACITY_SERVICE_NAME = "droboticflow-worker"
DEFAULT_GPU_CAPACITY_SERVICE_PORT = 3002
DEFAULT_API_PUBLIC_SERVICE_NAME = "droboticflow-api-public"
DEFAULT_AUTH_SECRET_NAME = "droboticflow-auth-secret"
DEFAULT_GPU_WORKFLOW_EXPECTED_LOG = "gpu request cannot be scheduled"
MANIFEST_FIXTURE_PATHS = {
"xplatform": ("dir", (("fixtures", "xplatform"), ("fixtures", "xplatformSourcePath"), ("xplatform", "sourcePath"))),
"csv": ("file", (("fixtures", "csv"), ("fixtures", "csvInputPath"))),
"json": ("file", (("fixtures", "json"), ("fixtures", "jsonInputPath"))),
"lerobot": ("dir", (("fixtures", "lerobot"), ("fixtures", "lerobotSourcePath"))),
"rosbag": ("any", (("fixtures", "rosbag"), ("fixtures", "rosbagSourcePath"))),
"model": ("file", (("fixtures", "model"), ("fixtures", "modelJsonPath"))),
}
MANIFEST_SHELL_COMMAND_PATHS = (("shellCommand",), ("shell", "command"))
@dataclass
class Check:
name: str
status: str
message: str
evidence: dict[str, Any]
required: bool = True
def env_bool(name: str, default: bool = False) -> bool:
raw = os.getenv(name, "").strip().lower()
if not raw:
return default
if raw in {"1", "true", "yes", "y", "on"}:
return True
if raw in {"0", "false", "no", "n", "off"}:
return False
raise SystemExit(f"{name} must be a boolean value")
def run_kubectl(args: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["kubectl", *args],
input=input_text,
text=True,
capture_output=True,
check=False,
)
def kubectl_json(args: list[str]) -> dict[str, Any] | None:
result = run_kubectl([*args, "-o", "json"])
if result.returncode != 0:
return None
try:
payload = json.loads(result.stdout)
except json.JSONDecodeError:
return None
return payload if isinstance(payload, dict) else None
def decode_secret_value(secret: dict[str, Any], *keys: str) -> str:
data = secret.get("data")
if not isinstance(data, dict):
return ""
for key in keys:
raw = data.get(key)
if not isinstance(raw, str) or not raw:
continue
try:
value = base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip()
except Exception:
continue
if value:
return value
return ""
def deployment_check(namespace: str, name: str, expected_tag: str) -> Check:
payload = kubectl_json(["-n", namespace, "get", "deploy", name])
if payload is None:
return Check("deployment:" + name, "failed", f"deployment {namespace}/{name} is missing or unreadable", {})
status = payload.get("status") if isinstance(payload.get("status"), dict) else {}
spec = payload.get("spec") if isinstance(payload.get("spec"), dict) else {}
template = spec.get("template") if isinstance(spec.get("template"), dict) else {}
pod_spec = template.get("spec") if isinstance(template.get("spec"), dict) else {}
containers = pod_spec.get("containers") if isinstance(pod_spec.get("containers"), list) else []
images = [item.get("image", "") for item in containers if isinstance(item, dict)]
ready = int(status.get("readyReplicas") or 0)
replicas = int(status.get("replicas") or spec.get("replicas") or 0)
issues: list[str] = []
if replicas <= 0 or ready < replicas:
issues.append(f"ready replicas {ready}/{replicas}")
if expected_tag and not all(str(image).endswith(":" + expected_tag) for image in images):
issues.append(f"image tag does not match {expected_tag}")
if issues:
return Check("deployment:" + name, "failed", "; ".join(issues), {"images": images, "readyReplicas": ready, "replicas": replicas})
return Check("deployment:" + name, "passed", "deployment is ready", {"images": images, "readyReplicas": ready, "replicas": replicas})
def runner_image_check(namespace: str, expected_tag: str) -> Check:
payload = kubectl_json(["-n", namespace, "get", "configmap", "droboticflow-config"])
if payload is None:
return Check("operator-runner-image", "failed", "configmap droboticflow-config is missing or unreadable", {})
data = payload.get("data") if isinstance(payload.get("data"), dict) else {}
image = str(data.get("OPERATOR_RUNNER_IMAGE") or "")
if not image:
return Check("operator-runner-image", "failed", "OPERATOR_RUNNER_IMAGE is not configured", {})
if expected_tag and not image.endswith(":" + expected_tag):
return Check("operator-runner-image", "failed", f"operator runner image tag does not match {expected_tag}", {"image": image})
return Check("operator-runner-image", "passed", "operator runner image is configured", {"image": image})
def configmap_data(namespace: str, name: str) -> dict[str, str]:
payload = kubectl_json(["-n", namespace, "get", "configmap", name])
if payload is None:
return {}
data = payload.get("data")
return {str(key): str(value) for key, value in data.items()} if isinstance(data, dict) else {}
def positive_int_env(name: str, default: int) -> int:
raw = os.getenv(name, "").strip()
if not raw:
return default
if not raw.isdigit() or int(raw) <= 0:
raise SystemExit(f"{name} must be a positive integer")
return int(raw)
def normalize_http_url(raw: str) -> str:
value = raw.strip().rstrip("/")
if not value:
return ""
parsed = urllib.parse.urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc or any(character.isspace() for character in value):
raise SystemExit(f"{raw} must be an absolute http(s) URL without whitespace")
return value
def is_private_load_balancer_host(host: str) -> bool:
try:
return ipaddress.ip_address(host).is_private
except ValueError:
return False
def platform_readiness_base_url(namespace: str) -> str:
for name in ("POC_PLATFORM_READINESS_API_BASE_URL", "API_BASE_URL", "BASE_URL"):
value = normalize_http_url(os.getenv(name, ""))
if value:
return value
service_name = (
os.getenv("POC_PLATFORM_READINESS_API_PUBLIC_SERVICE_NAME", "").strip()
or os.getenv("API_PUBLIC_SERVICE_NAME", "").strip()
or DEFAULT_API_PUBLIC_SERVICE_NAME
)
if not service_name or any(character.isspace() for character in service_name):
raise SystemExit("POC_PLATFORM_READINESS_API_PUBLIC_SERVICE_NAME must be non-empty and contain no whitespace")
payload = kubectl_json(["-n", namespace, "get", "svc", service_name])
if payload is None:
return ""
status = payload.get("status") if isinstance(payload.get("status"), dict) else {}
load_balancer = status.get("loadBalancer") if isinstance(status.get("loadBalancer"), dict) else {}
ingress = load_balancer.get("ingress") if isinstance(load_balancer.get("ingress"), list) else []
hosts = [str(item.get("ip") or item.get("hostname") or "").strip() for item in ingress if isinstance(item, dict)]
hosts = [host for host in hosts if host]
if not hosts:
return ""
for host in hosts:
if not is_private_load_balancer_host(host):
return f"http://{host}"
return f"http://{hosts[0]}"
def platform_readiness_token(namespace: str) -> tuple[str, dict[str, Any]]:
token = os.getenv("DROBOTICFLOW_API_TOKEN", "").strip()
if token:
return token, {"tokenSource": "env:DROBOTICFLOW_API_TOKEN"}
secret_name = os.getenv("POC_PLATFORM_READINESS_API_TOKEN_SECRET_NAME", DEFAULT_AUTH_SECRET_NAME).strip()
if not secret_name or any(character.isspace() for character in secret_name):
raise SystemExit("POC_PLATFORM_READINESS_API_TOKEN_SECRET_NAME must be non-empty and contain no whitespace")
secret = kubectl_json(["-n", namespace, "get", "secret", secret_name])
if secret is None:
return "", {"tokenSource": "secret", "secretName": secret_name}
token = decode_secret_value(secret, "DROBOTICFLOW_API_TOKEN", "API_TOKEN", "token")
return token, {"tokenSource": "secret", "secretName": secret_name}
def http_get_json(url: str, token: str, timeout_seconds: int) -> tuple[dict[str, Any] | None, str]:
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"})
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
try:
with opener.open(request, timeout=timeout_seconds) as response:
body = response.read().decode("utf-8")
except urllib.error.HTTPError as error:
return None, f"platform readiness API returned HTTP {error.code}"
except (OSError, urllib.error.URLError, TimeoutError) as error:
return None, f"platform readiness API request failed: {error}"
try:
payload = json.loads(body)
except json.JSONDecodeError as error:
return None, f"platform readiness API did not return JSON: {error.msg}"
return (payload, "") if isinstance(payload, dict) else (None, "platform readiness API payload is not a JSON object")
def gpu_capacity_pod_name() -> str:
raw = os.getenv("POC_GPU_CAPACITY_NAME", "").strip()
if raw:
return raw
return f"droboticflow-gpu-capacity-readiness-{int(time.time())}"
def gpu_capacity_probe_mode() -> str:
value = os.getenv("POC_GPU_CAPACITY_PROBE_MODE", "port-forward").strip().lower()
if value not in {"port-forward", "pod"}:
raise SystemExit("POC_GPU_CAPACITY_PROBE_MODE must be port-forward or pod")
return value
def endpoint_path_from_worker_url(raw: str) -> str:
value = raw.strip()
if not value:
return "/worker-api/capacity"
if value.startswith("/"):
return value
parsed = urllib.parse.urlsplit(value)
path = parsed.path or "/worker-api/capacity"
return path + (("?" + parsed.query) if parsed.query else "")
def free_local_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
def parse_capacity_payload(raw: str) -> tuple[dict[str, Any] | None, str]:
last_error = ""
for line in reversed(raw.splitlines()):
candidate = line.strip()
if not candidate.startswith("{"):
continue
try:
payload = json.loads(candidate)
except json.JSONDecodeError as error:
last_error = str(error)
continue
return (payload, "") if isinstance(payload, dict) else (None, "capacity payload is not a JSON object")
return None, f"capacity logs did not contain a JSON object{': ' + last_error if last_error else ''}"
def summarize_gpu_capacity(payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
gpu = payload.get("gpu")
if not isinstance(gpu, dict):
return {}, "capacity payload is missing gpu object"
nodes = gpu.get("nodes")
if not isinstance(nodes, list):
return {}, "gpu.nodes must be an array"
def int_value(name: str) -> int:
try:
return int(gpu.get(name) or 0)
except (TypeError, ValueError):
raise ValueError(f"gpu.{name} must be an integer-compatible value")
try:
schedulable_allocatable = int_value("schedulableAllocatable")
max_allocatable = int_value("maxAllocatable")
schedulable_node_count = int_value("schedulableNodeCount")
except ValueError as error:
return {}, str(error)
normalized_nodes: list[dict[str, Any]] = []
for item in nodes:
if not isinstance(item, dict):
continue
try:
allocatable = int(item.get("allocatable") or 0)
except (TypeError, ValueError):
return {}, "gpu.nodes[].allocatable must be an integer-compatible value"
normalized_nodes.append(
{
"name": str(item.get("name") or ""),
"allocatable": allocatable,
"selectorMatched": bool(item.get("selectorMatched")),
}
)
if not normalized_nodes:
return {}, "gpu.nodes did not contain any node objects"
evidence = {
"resourceName": str(gpu.get("resourceName") or ""),
"available": bool(gpu.get("available")),
"schedulableAllocatable": schedulable_allocatable,
"maxAllocatable": max_allocatable,
"schedulableNodeCount": schedulable_node_count,
"nodeCount": len(nodes),
"nodes": normalized_nodes,
}
if not evidence["resourceName"]:
return evidence, "gpu.resourceName is required"
if evidence["nodeCount"] <= 0:
return evidence, "gpu.nodes is empty; capacity endpoint did not report checked nodes"
return evidence, ""
def gpu_capacity_expectation_check(evidence: dict[str, Any], expected_available: bool) -> str:
available = bool(evidence.get("available"))
schedulable = int(evidence.get("schedulableAllocatable") or 0)
max_allocatable = int(evidence.get("maxAllocatable") or 0)
if expected_available:
if available and schedulable > 0 and max_allocatable > 0:
return ""
return (
"expected POC GPU capacity to be available, "
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}"
)
if not available and schedulable == 0 and max_allocatable == 0:
return ""
return (
"expected CPU-only POC GPU capacity, "
f"got available={available} schedulableAllocatable={schedulable} maxAllocatable={max_allocatable}"
)
def gpu_capacity_result_from_logs(raw: str, evidence_base: dict[str, Any], expected_available: bool) -> Check:
payload, parse_error = parse_capacity_payload(raw)
if payload is None:
return Check("gpu-capacity", "failed", parse_error, evidence_base)
capacity_evidence, payload_error = summarize_gpu_capacity(payload)
evidence = {**evidence_base, **capacity_evidence}
if payload_error:
return Check("gpu-capacity", "failed", payload_error, evidence)
expectation_error = gpu_capacity_expectation_check(evidence, expected_available)
if expectation_error:
return Check("gpu-capacity", "failed", expectation_error, evidence)
message = "GPU capacity is available as expected" if expected_available else "POC GPU capacity is CPU-only as expected"
return Check("gpu-capacity", "passed", message, evidence)
def gpu_capacity_pod_check(
namespace: str,
data: dict[str, str],
worker_url: str,
expected_available: bool,
timeout_seconds: int,
http_timeout_seconds: int,
) -> Check:
image = os.getenv("POC_GPU_CAPACITY_IMAGE", "").strip() or os.getenv("OPERATOR_RUNNER_IMAGE", "").strip() or data.get("OPERATOR_RUNNER_IMAGE", "")
if not image:
return Check("gpu-capacity", "failed", "OPERATOR_RUNNER_IMAGE is not configured for GPU capacity readiness", {})
if any(character.isspace() for character in image):
return Check("gpu-capacity", "failed", "POC_GPU_CAPACITY_IMAGE must not contain whitespace", {"image": image})
image_pull_secret = os.getenv("K8S_IMAGE_PULL_SECRET_NAME", "").strip() or data.get("K8S_IMAGE_PULL_SECRET_NAME", "")
pod_name = gpu_capacity_pod_name()
manifest: dict[str, Any] = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": pod_name,
"namespace": namespace,
"labels": {"app": "droboticflow", "component": "poc-gpu-capacity-readiness"},
},
"spec": {
"restartPolicy": "Never",
"containers": [
{
"name": "capacity",
"image": image,
"imagePullPolicy": "IfNotPresent",
"command": [
"python",
"-c",
"\n".join(
[
"import json",
"import os",
"import urllib.request",
'url = os.environ["WORKER_CAPACITY_URL"]',
'timeout = int(os.environ.get("WORKER_CAPACITY_TIMEOUT_SECONDS", "30"))',
"with urllib.request.urlopen(url, timeout=timeout) as response:",
' body = response.read().decode("utf-8")',
"json.loads(body)",
"print(body)",
]
),
],
"env": [
{"name": "WORKER_CAPACITY_URL", "value": worker_url},
{"name": "WORKER_CAPACITY_TIMEOUT_SECONDS", "value": str(http_timeout_seconds)},
],
}
],
},
}
if image_pull_secret:
manifest["spec"]["imagePullSecrets"] = [{"name": image_pull_secret}]
evidence_base = {"probeMode": "pod", "podName": pod_name, "image": image, "workerUrl": worker_url, "expectedAvailable": expected_available}
applied = run_kubectl(["apply", "-f", "-"], input_text=json.dumps(manifest))
if applied.returncode != 0:
return Check("gpu-capacity", "failed", "failed to create GPU capacity readiness Pod", {**evidence_base, "stderr": applied.stderr.strip()})
try:
deadline = time.time() + timeout_seconds
phase = ""
while time.time() < deadline:
result = run_kubectl(["-n", namespace, "get", "pod", pod_name, "-o", "jsonpath={.status.phase}"])
phase = result.stdout.strip() if result.returncode == 0 else ""
if phase in {"Succeeded", "Failed"}:
break
time.sleep(2)
if phase not in {"Succeeded", "Failed"}:
return Check("gpu-capacity", "failed", "timed out waiting for GPU capacity readiness Pod", {**evidence_base, "phase": phase or "unknown"})
logs_result = run_kubectl(["-n", namespace, "logs", pod_name])
logs = logs_result.stdout if logs_result.returncode == 0 else logs_result.stderr
if phase != "Succeeded":
return Check("gpu-capacity", "failed", "GPU capacity readiness Pod failed", {**evidence_base, "phase": phase, "logs": logs.strip()[-400:]})
return gpu_capacity_result_from_logs(logs, evidence_base, expected_available)
finally:
run_kubectl(["-n", namespace, "delete", "pod", pod_name, "--ignore-not-found"])
def gpu_capacity_port_forward_check(
namespace: str,
worker_url: str,
expected_available: bool,
timeout_seconds: int,
http_timeout_seconds: int,
) -> Check:
service_name = os.getenv("POC_GPU_CAPACITY_SERVICE_NAME", DEFAULT_GPU_CAPACITY_SERVICE_NAME).strip()
service_port = positive_int_env("POC_GPU_CAPACITY_SERVICE_PORT", DEFAULT_GPU_CAPACITY_SERVICE_PORT)
if not service_name or any(character.isspace() for character in service_name):
return Check("gpu-capacity", "failed", "POC_GPU_CAPACITY_SERVICE_NAME must be non-empty and contain no whitespace", {})
local_port = free_local_port()
endpoint_path = endpoint_path_from_worker_url(worker_url)
local_url = f"http://127.0.0.1:{local_port}{endpoint_path}"
evidence_base = {
"probeMode": "port-forward",
"serviceName": service_name,
"servicePort": service_port,
"localPort": local_port,
"workerUrl": local_url,
"expectedAvailable": expected_available,
}
process = subprocess.Popen(
["kubectl", "-n", namespace, "port-forward", f"service/{service_name}", f"{local_port}:{service_port}"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
try:
deadline = time.time() + timeout_seconds
last_error = ""
while time.time() < deadline:
if process.poll() is not None:
stdout, stderr = process.communicate()
return Check(
"gpu-capacity",
"failed",
"kubectl port-forward for GPU capacity exited early",
{**evidence_base, "stdout": stdout.strip()[-400:], "stderr": stderr.strip()[-400:]},
)
try:
with opener.open(local_url, timeout=http_timeout_seconds) as response:
logs = response.read().decode("utf-8")
return gpu_capacity_result_from_logs(logs, evidence_base, expected_available)
except (OSError, urllib.error.URLError, TimeoutError) as error:
last_error = str(error)
time.sleep(0.25)
return Check("gpu-capacity", "failed", "timed out waiting for GPU capacity port-forward", {**evidence_base, "lastError": last_error})
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=3)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=3)
def gpu_capacity_check(namespace: str) -> Check:
expected_available = env_bool("POC_GPU_CAPACITY_EXPECT_AVAILABLE", True)
probe_mode = gpu_capacity_probe_mode()
timeout_seconds = positive_int_env("POC_GPU_CAPACITY_TIMEOUT_SECONDS", 120)
http_timeout_seconds = positive_int_env("POC_GPU_CAPACITY_HTTP_TIMEOUT_SECONDS", 30)
worker_url = os.getenv("POC_GPU_CAPACITY_WORKER_URL", DEFAULT_GPU_CAPACITY_WORKER_URL).strip()
if not worker_url or any(character.isspace() for character in worker_url):
return Check("gpu-capacity", "failed", "POC_GPU_CAPACITY_WORKER_URL must be a non-empty URL without whitespace", {})
data = configmap_data(namespace, "droboticflow-config")
if probe_mode == "pod":
return gpu_capacity_pod_check(namespace, data, worker_url, expected_available, timeout_seconds, http_timeout_seconds)
return gpu_capacity_port_forward_check(namespace, worker_url, expected_available, timeout_seconds, http_timeout_seconds)
def platform_gpu_workflow_preflight_check(namespace: str, expected_gpu_available: bool) -> Check:
enabled = env_bool("POC_PLATFORM_READINESS_ENABLED", True)
if not enabled:
return Check(
"platform-gpu-workflow-preflight",
"skipped",
"platform GPU workflow preflight readiness is disabled",
{"enabled": False},
required=False,
)
base_url = platform_readiness_base_url(namespace)
if not base_url:
return Check(
"platform-gpu-workflow-preflight",
"failed",
"platform readiness API base URL is unavailable",
{
"serviceName": os.getenv("POC_PLATFORM_READINESS_API_PUBLIC_SERVICE_NAME", "")
or os.getenv("API_PUBLIC_SERVICE_NAME", "")
or DEFAULT_API_PUBLIC_SERVICE_NAME
},
)
token, token_evidence = platform_readiness_token(namespace)
if not token:
return Check("platform-gpu-workflow-preflight", "failed", "DROBOTICFLOW_API_TOKEN is unavailable for platform readiness API", token_evidence)
timeout_seconds = positive_int_env("POC_PLATFORM_READINESS_HTTP_TIMEOUT_SECONDS", 20)
payload, error = http_get_json(f"{base_url}/api/acceptance/poc-readiness", token, timeout_seconds)
evidence: dict[str, Any] = {**token_evidence, "apiBaseUrl": base_url, "expectedGpuAvailable": expected_gpu_available}
if error:
return Check("platform-gpu-workflow-preflight", "failed", error, evidence)
assert payload is not None
gpu_gate = payload.get("gpu") if isinstance(payload.get("gpu"), dict) else {}
workflow_gate = payload.get("gpuWorkflowPreflight") if isinstance(payload.get("gpuWorkflowPreflight"), dict) else {}
workflow_evidence = workflow_gate.get("evidence") if isinstance(workflow_gate.get("evidence"), dict) else {}
workflow_run_status = workflow_evidence.get("runStatus") or workflow_evidence.get("status")
evidence.update(
{
"platformStatus": payload.get("status"),
"gpuStatus": gpu_gate.get("status"),
"gpuWorkflowPreflightStatus": workflow_gate.get("status"),
"runId": workflow_evidence.get("runId"),
"runStatus": workflow_run_status,
"expectedLogSubstring": workflow_evidence.get("expectedLogSubstring"),
}
)
if expected_gpu_available:
if gpu_gate.get("status") != "passed":
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU hardware gate is not passed while GPU capacity is expected", evidence)
if workflow_gate.get("status") != "passed":
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow gate is not passed while GPU capacity is expected", evidence)
if not workflow_evidence.get("runId"):
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow gate is missing runId evidence", evidence)
if workflow_run_status != "completed":
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow gate must record a completed run while GPU capacity is expected", evidence)
return Check("platform-gpu-workflow-preflight", "passed", "platform GPU workflow completion evidence is ready", evidence)
if workflow_gate.get("status") != "passed":
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight gate is not passed", evidence)
if not workflow_evidence.get("runId"):
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight is missing runId evidence", evidence)
if workflow_run_status != "failed":
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight must record a failed run while POC is CPU-only", evidence)
expected_log = os.getenv("POC_GPU_WORKFLOW_EXPECT_LOG_SUBSTRING", DEFAULT_GPU_WORKFLOW_EXPECTED_LOG).strip()
recorded_log = str(workflow_evidence.get("expectedLogSubstring") or "")
if expected_log and expected_log not in recorded_log:
return Check("platform-gpu-workflow-preflight", "failed", "platform GPU workflow preflight log evidence does not match the expected blocker", evidence)
return Check("platform-gpu-workflow-preflight", "passed", "platform GPU workflow preflight evidence is ready", evidence)
def api_pod_name(namespace: str) -> str:
result = run_kubectl(["-n", namespace, "get", "pod", "-l", "app=droboticflow-api", "-o", "jsonpath={.items[0].metadata.name}"])
return result.stdout.strip() if result.returncode == 0 else ""
def normalize_workspace_mount_path(raw: str) -> str:
value = raw.strip().rstrip("/")
return value or DEFAULT_WORKSPACE_MOUNT_PATH
def default_fixture_manifest_path(workspace_mount_path: str) -> str:
return f"{workspace_mount_path}/{DEFAULT_FIXTURE_MANIFEST_RELATIVE_PATH}"
def is_under_workspace(path: str, workspace_mount_path: str) -> bool:
return path == workspace_mount_path or path.startswith(workspace_mount_path + "/")
def workspace_path_check(namespace: str, workspace_mount_path: str, path: str, kind: str, label: str) -> Check:
if not is_under_workspace(path, workspace_mount_path):
return Check(label, "failed", f"{label} must point under WORKSPACE_MOUNT_PATH {workspace_mount_path}", {"path": path, "workspaceMountPath": workspace_mount_path})
pod = api_pod_name(namespace)
if not pod:
return Check(label, "failed", "no droboticflow-api pod is available for workspace validation", {"path": path})
test_script = {
"file": 'test -f "$1" && test -r "$1" && test -s "$1"',
"dir": 'test -d "$1" && test -r "$1"',
"any": 'test -e "$1" && test -r "$1"',
}[kind]
result = run_kubectl(["-n", namespace, "exec", pod, "--", "sh", "-c", test_script, "--", path])
if result.returncode != 0:
detail = "readable non-empty file" if kind == "file" else f"readable {kind}"
return Check(label, "failed", f"{label} is not a {detail} inside the shared workspace PVC", {"path": path})
return Check(label, "passed", f"{label} is readable in the shared workspace PVC", {"path": path})
def fixture_manifest_check(namespace: str, workspace_mount_path: str, manifest_path: str) -> Check:
file_check = workspace_path_check(namespace, workspace_mount_path, manifest_path, "file", "fixture-manifest")
if file_check.status != "passed":
return file_check
content = read_workspace_text(namespace, workspace_mount_path, manifest_path, "fixture-manifest")
if content is None:
return Check("fixture-manifest", "failed", "fixture-manifest could not be read from the shared workspace PVC", {"path": manifest_path})
try:
payload = json.loads(content)
except json.JSONDecodeError as error:
return Check(
"fixture-manifest",
"failed",
f"fixture-manifest is not valid JSON: {error.msg}",
{"path": manifest_path, "line": error.lineno, "column": error.colno},
)
if not isinstance(payload, dict):
return Check("fixture-manifest", "failed", "fixture-manifest must be a JSON object", {"path": manifest_path})
missing: list[str] = []
path_errors: list[dict[str, Any]] = []
fixtures: dict[str, str] = {}
for name, (kind, paths) in MANIFEST_FIXTURE_PATHS.items():
value = manifest_config_text(payload, paths)
if not value:
missing.append(name)
continue
fixtures[name] = value
check = workspace_path_check(namespace, workspace_mount_path, value, kind, "fixture:" + name)
if check.status != "passed":
path_errors.append({"name": name, "path": value, "message": check.message})
shell_command = manifest_config_text(payload, MANIFEST_SHELL_COMMAND_PATHS)
if not shell_command:
missing.append("shellCommand")
if missing or path_errors:
details: list[str] = []
if missing:
details.append("missing fields: " + ", ".join(missing))
if path_errors:
details.append("unready paths: " + "; ".join(f"{item['name']}={item['path']}" for item in path_errors))
return Check(
"fixture-manifest",
"failed",
"fixture-manifest is incomplete: " + "; ".join(details),
{
"path": manifest_path,
"fixtures": fixtures,
"missing": missing,
"pathErrors": path_errors,
"shellCommandConfigured": bool(shell_command),
},
)
return Check(
"fixture-manifest",
"passed",
"fixture-manifest and real fixture paths are ready",
{"path": manifest_path, "fixtures": fixtures, "shellCommandConfigured": True},
)
def read_workspace_text(namespace: str, workspace_mount_path: str, path: str, label: str) -> str | None:
if not is_under_workspace(path, workspace_mount_path):
return None
pod = api_pod_name(namespace)
if not pod:
return None
result = run_kubectl(["-n", namespace, "exec", pod, "--", "cat", path])
if result.returncode != 0:
return None
return result.stdout
def manifest_nested_value(payload: dict[str, Any], path: tuple[str, ...]) -> Any:
current: Any = payload
for key in path:
if not isinstance(current, dict) or key not in current:
return None
current = current[key]
return current
def manifest_config_text(payload: dict[str, Any], paths: tuple[tuple[str, ...], ...]) -> str:
for path in paths:
value = manifest_nested_value(payload, path)
if value is None:
continue
if isinstance(value, str):
return value.strip()
return ""
return ""
def required_external_enabled(target: str) -> bool:
raw = os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", "bos,mongo")
return target in {part.strip().lower() for part in raw.split(",") if part.strip()}
def main() -> None:
namespace = os.getenv("K8S_NAMESPACE", DEFAULT_NAMESPACE)
expected_tag = os.getenv("IMAGE_TAG", "").strip()
workspace_mount_path = normalize_workspace_mount_path(os.getenv("WORKSPACE_MOUNT_PATH", DEFAULT_WORKSPACE_MOUNT_PATH))
manifest_path = os.getenv("POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH", "").strip() or default_fixture_manifest_path(workspace_mount_path)
allow_generated = env_bool("POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", False)
expected_gpu_available = env_bool("POC_GPU_CAPACITY_EXPECT_AVAILABLE", True)
checks = [
deployment_check(namespace, "droboticflow-api", expected_tag),
deployment_check(namespace, "droboticflow-worker", expected_tag),
runner_image_check(namespace, expected_tag),
gpu_capacity_check(namespace),
platform_gpu_workflow_preflight_check(namespace, expected_gpu_available),
]
if allow_generated:
checks.append(
Check(
"fixture-manifest",
"failed",
"generated fixtures are enabled for diagnostics; strict POC readiness requires real fixture evidence",
{"allowGeneratedFixtures": True},
required=True,
)
)
else:
checks.append(fixture_manifest_check(namespace, workspace_mount_path, manifest_path))
failed_required = [check for check in checks if check.required and check.status == "failed"]
payload = {
"status": "ready" if not failed_required else "blocked",
"namespace": namespace,
"workspaceMountPath": workspace_mount_path,
"expectedImageTag": expected_tag,
"requiredExternals": sorted({part.strip().lower() for part in os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", "bos,mongo").split(",") if part.strip()}),
"checks": [check.__dict__ for check in checks],
"missing": [{"name": check.name, "message": check.message, "evidence": check.evidence} for check in failed_required],
"nextActions": [],
}
if any(check.name == "fixture-manifest" and check.status == "failed" for check in checks):
if allow_generated:
payload["nextActions"].append(
"Unset POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES and create a real fixture manifest with npm run acceptance:poc-fixtures using non-generated local or /workspace fixture data."
)
else:
payload["nextActions"].append("Create a real fixture manifest with npm run acceptance:poc-fixtures using non-generated local or /workspace fixture data.")
if any(check.name == "platform-gpu-workflow-preflight" and check.status == "failed" for check in checks):
payload["nextActions"].append("Run npm run acceptance:poc-gpu-workflow with a real DROBOTICFLOW_API_TOKEN, then rerun npm run acceptance:poc-readiness.")
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
raise SystemExit(0 if payload["status"] == "ready" else 1)
if __name__ == "__main__":
main()