droboticflow/scripts/discover_poc_acceptance_fixtures.py
2026-07-07 18:57:49 +08:00

541 lines
23 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import os
import pathlib
import subprocess
from dataclasses import dataclass
from typing import Any
DEFAULT_NAMESPACE = "droboticflow"
DEFAULT_WORKSPACE = "/workspace"
DEFAULT_MANIFEST_PATH = "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"
DEFAULT_DISCOVERY_ROOTS = (
"/workspace/datasets/fixtures",
"/workspace/datasets",
"/workspace/source-data",
"/workspace/imports",
"/workspace/raw",
"/workspace",
)
DEFAULT_EXCLUDE_ROOTS = (
"/workspace/system/acceptance/general",
"/workspace/system/acceptance/droboticflow",
"/workspace/system/builds/contexts",
"/workspace/system/builds/images",
"/workspace/poc-workflow-acceptance-*",
"/workspace/system/runs",
)
MANIFEST_FIXTURE_PATHS = {
"xplatform": ("dir", (("fixtures", "xplatform"), ("fixtures", "xplatformSourcePath"), ("xplatform", "sourcePath"))),
"csv": ("file", (("fixtures", "csv"), ("fixtures", "csvInputPath"))),
"json": ("file", (("fixtures", "json"), ("fixtures", "jsonInputPath"))),
"lerobot": ("dir", (("fixtures", "lerobot"), ("fixtures", "lerobotSourcePath"))),
"rosbag": ("any", (("fixtures", "rosbag"), ("fixtures", "rosbagSourcePath"))),
"model": ("file", (("fixtures", "model"), ("fixtures", "modelJsonPath"))),
}
MANIFEST_SHELL_COMMAND_PATHS = (("shellCommand",), ("shell", "command"))
JSON_READ_LIMIT_BYTES = 2 * 1024 * 1024
@dataclass(frozen=True)
class Candidate:
path: str
score: int
evidence: dict[str, Any]
def run_kubectl(args: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
return subprocess.run(["kubectl", *args], input=input_text, text=True, capture_output=True, check=False)
def api_pod_name(namespace: str) -> str:
result = run_kubectl(["-n", namespace, "get", "pod", "-l", "app=droboticflow-api", "-o", "jsonpath={.items[0].metadata.name}"])
return result.stdout.strip() if result.returncode == 0 else ""
def workspace_path(value: str, workspace_mount: str) -> bool:
value = value.rstrip("/") or "/"
workspace_mount = workspace_mount.rstrip("/") or "/"
return value == workspace_mount or value.startswith(workspace_mount + "/")
def normalize_roots(raw: str, workspace_mount: str, *, label: str = "discovery root") -> list[str]:
values = [item.strip().rstrip("/") for item in raw.split(",") if item.strip()]
roots: list[str] = []
seen: set[str] = set()
for value in values:
if not value.startswith("/"):
raise SystemExit(f"{label} must be absolute: {value}")
if not workspace_path(value, workspace_mount):
raise SystemExit(f"{label} must point under {workspace_mount}: {value}")
if value not in seen:
roots.append(value)
seen.add(value)
return roots
def default_exclude_roots(workspace_mount: str, include_acceptance_output: bool) -> list[str]:
workspace_mount = workspace_mount.rstrip("/") or "/"
roots = [workspace_mount + item[len(DEFAULT_WORKSPACE) :] for item in DEFAULT_EXCLUDE_ROOTS]
if include_acceptance_output:
roots = [
item
for item in roots
if item
not in {
workspace_mount + "/system/acceptance/general",
workspace_mount + "/system/acceptance/droboticflow",
}
]
return roots
def find_prune_clause(exclude_root_count: int, first_arg_index: int = 2) -> str:
if exclude_root_count <= 0:
return ""
terms: list[str] = []
for index in range(first_arg_index, first_arg_index + exclude_root_count):
terms.append(f'-path "${index}" -o -path "${index}/*"')
return "\\( " + " -o ".join(terms) + " \\) -prune -o "
def remote_shell(namespace: str, pod: str, script: str, args: list[str]) -> str:
result = run_kubectl(["-n", namespace, "exec", pod, "--", "sh", "-c", script, "--", *args])
if result.returncode != 0:
return ""
return result.stdout
def remote_text(namespace: str, pod: str, path: str, limit: int = JSON_READ_LIMIT_BYTES) -> str:
result = run_kubectl(
["-n", namespace, "exec", pod, "--", "sh", "-c", 'head -c "$2" "$1" 2>/dev/null', "--", path, str(limit)]
)
if result.returncode != 0:
return ""
return result.stdout
def write_remote_text(namespace: str, pod: str, path: str, text: str) -> bool:
result = run_kubectl(
[
"-n",
namespace,
"exec",
"-i",
pod,
"--",
"sh",
"-c",
'mkdir -p "$(dirname "$1")" && cat > "$1" && test -s "$1"',
"--",
path,
],
input_text=text,
)
return result.returncode == 0
def find_paths(namespace: str, pod: str, root: str, exclude_roots: list[str], expression: str, limit: int) -> list[str]:
limit_arg_index = 2 + len(exclude_roots)
prune_clause = find_prune_clause(len(exclude_roots))
script = (
'if [ ! -d "$1" ]; then exit 0; fi; '
f'find "$1" {prune_clause}{expression} -print | sort | head -n "${limit_arg_index}"'
)
output = remote_shell(namespace, pod, script, [root, *exclude_roots, str(limit)])
paths = [line.strip() for line in output.splitlines() if line.strip()]
return paths
def sorted_candidates(candidates: list[Candidate]) -> list[Candidate]:
return sorted(candidates, key=lambda item: (-item.score, item.path))
def first_video_count(namespace: str, pod: str, root: str, exclude_roots: list[str]) -> int:
paths = find_paths(
namespace,
pod,
root,
exclude_roots,
"\\( -type f \\( -iname '*.mp4' -o -iname '*.mov' -o -iname '*.avi' -o -iname '*.mkv' \\) \\)",
20,
)
return len(paths)
def discover_xplatform(namespace: str, pod: str, roots: list[str], exclude_roots: list[str], limit: int) -> list[Candidate]:
candidates: list[Candidate] = []
limit_arg_index = 2 + len(exclude_roots)
prune_clause = find_prune_clause(len(exclude_roots))
script = (
'if [ ! -d "$1" ]; then exit 0; fi; '
f'find "$1" {prune_clause}-type f -name meta.json -print '
"| while IFS= read -r file; do dir=$(dirname \"$file\"); "
f'if [ -s "$dir/video_meta.json" ]; then printf "%s\\n" "$dir"; fi; done | sort | head -n "${limit_arg_index}"'
)
for root in roots:
output = remote_shell(namespace, pod, script, [root, *exclude_roots, str(limit)])
for path in output.splitlines():
path = path.strip()
if not path:
continue
video_count = first_video_count(namespace, pod, path, exclude_roots)
score = 100 + min(video_count, 20)
candidates.append(
Candidate(path=path, score=score, evidence={"metaJson": path + "/meta.json", "videoMetaJson": path + "/video_meta.json", "videoCountSample": video_count})
)
return sorted_candidates(dedupe_candidates(candidates))[:limit]
def discover_lerobot(namespace: str, pod: str, roots: list[str], exclude_roots: list[str], limit: int) -> list[Candidate]:
candidates: list[Candidate] = []
limit_arg_index = 2 + len(exclude_roots)
prune_clause = find_prune_clause(len(exclude_roots))
script = (
'if [ ! -d "$1" ]; then exit 0; fi; '
f'find "$1" {prune_clause}-type f -path "*/meta/info.json" -print '
"| while IFS= read -r file; do dir=$(dirname \"$(dirname \"$file\")\"); "
f'if [ -d "$dir/data" ] || [ -d "$dir/videos" ]; then printf "%s\\n" "$dir"; fi; done | sort | head -n "${limit_arg_index}"'
)
for root in roots:
output = remote_shell(namespace, pod, script, [root, *exclude_roots, str(limit)])
for path in output.splitlines():
path = path.strip()
if path:
candidates.append(Candidate(path=path, score=100, evidence={"infoJson": path + "/meta/info.json"}))
return sorted_candidates(dedupe_candidates(candidates))[:limit]
def discover_rosbag(namespace: str, pod: str, roots: list[str], exclude_roots: list[str], limit: int) -> list[Candidate]:
candidates: list[Candidate] = []
for root in roots:
files = find_paths(
namespace,
pod,
root,
exclude_roots,
"\\( -type f \\( -iname '*.bag' -o -iname '*.mcap' \\) \\)",
limit,
)
for path in files:
candidates.append(Candidate(path=path, score=100, evidence={"format": pathlib.PurePosixPath(path).suffix.lower().lstrip(".")}))
metadata = find_paths(namespace, pod, root, exclude_roots, "\\( -type f -name metadata.yaml \\)", limit)
for path in metadata:
directory = str(pathlib.PurePosixPath(path).parent)
candidates.append(Candidate(path=directory, score=90, evidence={"metadataYaml": path, "format": "rosbag2"}))
return sorted_candidates(dedupe_candidates(candidates))[:limit]
def parse_json_candidate(namespace: str, pod: str, path: str) -> Any | None:
text = remote_text(namespace, pod, path)
if not text.strip():
return None
try:
return json.loads(text)
except json.JSONDecodeError:
return None
def json_input_score(path: str, payload: Any) -> int:
name = pathlib.PurePosixPath(path).name.lower()
score = 1
if isinstance(payload, list) and payload and all(isinstance(item, dict) for item in payload):
score += 50
if isinstance(payload, dict):
items = payload.get("items")
if isinstance(items, list) and items and all(isinstance(item, dict) for item in items):
score += 60
elif payload:
score += 20
if any(token in name for token in ("record", "input", "item", "data", "sample")):
score += 20
if name in {"meta.json", "video_meta.json", "model.json", "operator.json", "repository.json", "poc-operator-acceptance.json"}:
score -= 80
return score
def model_score(path: str, payload: Any) -> int:
if not isinstance(payload, dict) or not str(payload.get("type") or "").strip():
return -100
name = pathlib.PurePosixPath(path).name.lower()
score = 50
if "model" in name:
score += 50
if any(key in payload for key in ("weights", "threshold", "positiveLabel", "negativeLabel", "features")):
score += 25
return score
def discover_json_files(
namespace: str,
pod: str,
roots: list[str],
exclude_roots: list[str],
limit: int,
) -> tuple[list[Candidate], list[Candidate]]:
json_candidates: list[Candidate] = []
model_candidates: list[Candidate] = []
seen: set[str] = set()
for root in roots:
paths = find_paths(namespace, pod, root, exclude_roots, "\\( -type f -iname '*.json' -size +0c \\)", limit * 8)
for path in paths:
if path in seen:
continue
seen.add(path)
payload = parse_json_candidate(namespace, pod, path)
if payload is None:
continue
input_score = json_input_score(path, payload)
if input_score > 0:
json_candidates.append(Candidate(path=path, score=input_score, evidence=json_payload_evidence(payload)))
candidate_model_score = model_score(path, payload)
if candidate_model_score > 0:
model_candidates.append(Candidate(path=path, score=candidate_model_score, evidence={"type": str(payload.get("type") or ""), "keys": sorted(payload.keys())[:12]}))
return sorted_candidates(json_candidates)[:limit], sorted_candidates(model_candidates)[:limit]
def json_payload_evidence(payload: Any) -> dict[str, Any]:
if isinstance(payload, list):
return {"shape": "array", "itemCount": len(payload)}
if isinstance(payload, dict):
items = payload.get("items")
return {"shape": "object", "keys": sorted(payload.keys())[:12], "itemCount": len(items) if isinstance(items, list) else None}
return {"shape": type(payload).__name__}
def manifest_nested_value(payload: dict[str, Any], path: tuple[str, ...]) -> Any:
current: Any = payload
for key in path:
if not isinstance(current, dict) or key not in current:
return None
current = current[key]
return current
def manifest_config_text(payload: dict[str, Any], paths: tuple[tuple[str, ...], ...]) -> str:
for path in paths:
value = manifest_nested_value(payload, path)
if value is None:
continue
if isinstance(value, str):
return value.strip()
return ""
return ""
def load_existing_manifest(namespace: str, pod: str, path: str, workspace_mount: str) -> dict[str, Any] | None:
if not path or not workspace_path(path, workspace_mount):
return None
payload = parse_json_candidate(namespace, pod, path)
if isinstance(payload, dict):
return payload
return None
def remote_fixture_ready(namespace: str, pod: str, path: str, kind: str, workspace_mount: str) -> bool:
if not path or not workspace_path(path, workspace_mount):
return False
test_script = {
"file": 'test -f "$1" && test -r "$1" && test -s "$1"',
"dir": 'test -d "$1" && test -r "$1"',
"any": 'test -e "$1" && test -r "$1"',
}[kind]
result = run_kubectl(["-n", namespace, "exec", pod, "--", "sh", "-c", test_script, "--", path])
return result.returncode == 0
def manifest_candidates(
namespace: str,
pod: str,
payload: dict[str, Any] | None,
manifest_path: str,
workspace_mount: str,
) -> dict[str, Candidate]:
if not payload:
return {}
selected: dict[str, Candidate] = {}
for name, (kind, paths) in MANIFEST_FIXTURE_PATHS.items():
value = manifest_config_text(payload, paths)
if not value:
continue
if not remote_fixture_ready(namespace, pod, value, kind, workspace_mount):
continue
selected[name] = Candidate(
path=value,
score=10_000,
evidence={"source": "manifest", "manifestPath": manifest_path},
)
return selected
def prepend_manifest_candidate(candidates: list[Candidate], manifest_candidate: Candidate | None) -> list[Candidate]:
if manifest_candidate is None:
return candidates
merged = [manifest_candidate, *candidates]
return sorted_candidates(dedupe_candidates(merged))
def discover_csv(namespace: str, pod: str, roots: list[str], exclude_roots: list[str], limit: int) -> list[Candidate]:
candidates: list[Candidate] = []
for root in roots:
paths = find_paths(
namespace,
pod,
root,
exclude_roots,
"\\( -type f \\( -iname '*.csv' -o -iname '*.tsv' \\) -size +0c \\)",
limit,
)
for path in paths:
header = remote_text(namespace, pod, path, 4096)
lines = [line for line in header.splitlines() if line.strip()]
delimiter = "\t" if pathlib.PurePosixPath(path).suffix.lower() == ".tsv" else ","
score = 50
if len(lines) >= 2:
score += 40
if lines and delimiter in lines[0]:
score += 20
candidates.append(Candidate(path=path, score=score, evidence={"lineSampleCount": len(lines), "delimiter": delimiter}))
return sorted_candidates(dedupe_candidates(candidates))[:limit]
def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]:
best: dict[str, Candidate] = {}
for candidate in candidates:
previous = best.get(candidate.path)
if previous is None or candidate.score > previous.score:
best[candidate.path] = candidate
return list(best.values())
def candidate_to_json(candidate: Candidate) -> dict[str, Any]:
return {"path": candidate.path, "score": candidate.score, "evidence": candidate.evidence}
def build_manifest(selected: dict[str, Candidate], shell_command: str) -> dict[str, Any]:
return {
"fixtures": {
"xplatform": selected["xplatform"].path,
"csv": selected["csv"].path,
"json": selected["json"].path,
"lerobot": selected["lerobot"].path,
"rosbag": selected["rosbag"].path,
"model": selected["model"].path,
},
"shellCommand": shell_command,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Discover existing real POC acceptance fixture candidates on the shared workspace PVC")
parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or DEFAULT_NAMESPACE)
parser.add_argument("--workspace-mount-path", default=os.getenv("WORKSPACE_MOUNT_PATH") or DEFAULT_WORKSPACE)
parser.add_argument("--roots", default=os.getenv("POC_ACCEPTANCE_DISCOVERY_ROOTS") or ",".join(DEFAULT_DISCOVERY_ROOTS))
parser.add_argument("--exclude-roots", default=os.getenv("POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS") or "")
parser.add_argument("--manifest-path", default=os.getenv("POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH") or DEFAULT_MANIFEST_PATH)
parser.add_argument("--shell-command", default=os.getenv("POC_ACCEPTANCE_SHELL_COMMAND") or "")
parser.add_argument("--max-results", type=int, default=int(os.getenv("POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS") or "10"))
parser.add_argument("--include-acceptance-output", action="store_true", default=os.getenv("POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT", "").lower() in {"1", "true", "yes", "on"})
parser.add_argument("--write-manifest", action="store_true", help="write the selected manifest to --manifest-path when every required real fixture is found")
parser.add_argument("--require-complete", action="store_true", help="exit non-zero when the discovery result is incomplete")
args = parser.parse_args()
if args.max_results <= 0:
raise SystemExit("--max-results must be positive")
workspace_mount = args.workspace_mount_path.rstrip("/") or "/"
if not args.manifest_path.startswith("/"):
raise SystemExit("--manifest-path must be absolute")
if not workspace_path(args.manifest_path, workspace_mount):
raise SystemExit(f"--manifest-path must point under {workspace_mount}: {args.manifest_path}")
roots = normalize_roots(args.roots, workspace_mount)
exclude_roots_raw = args.exclude_roots.strip()
if not exclude_roots_raw:
exclude_roots_raw = ",".join(default_exclude_roots(workspace_mount, args.include_acceptance_output))
exclude_roots = normalize_roots(exclude_roots_raw, workspace_mount, label="discovery exclude root")
pod = api_pod_name(args.namespace)
if not pod:
payload = {
"status": "error",
"namespace": args.namespace,
"message": "no droboticflow-api pod is available for workspace fixture discovery",
}
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
return 2
xplatform = discover_xplatform(args.namespace, pod, roots, exclude_roots, args.max_results)
lerobot = discover_lerobot(args.namespace, pod, roots, exclude_roots, args.max_results)
rosbag = discover_rosbag(args.namespace, pod, roots, exclude_roots, args.max_results)
csv = discover_csv(args.namespace, pod, roots, exclude_roots, args.max_results)
json_input, model = discover_json_files(args.namespace, pod, roots, exclude_roots, args.max_results)
existing_manifest = load_existing_manifest(args.namespace, pod, args.manifest_path, workspace_mount)
manifest_selected = manifest_candidates(args.namespace, pod, existing_manifest, args.manifest_path, workspace_mount)
candidates = {
"xplatform": prepend_manifest_candidate(xplatform, manifest_selected.get("xplatform")),
"csv": prepend_manifest_candidate(csv, manifest_selected.get("csv")),
"json": prepend_manifest_candidate(json_input, manifest_selected.get("json")),
"lerobot": prepend_manifest_candidate(lerobot, manifest_selected.get("lerobot")),
"rosbag": prepend_manifest_candidate(rosbag, manifest_selected.get("rosbag")),
"model": prepend_manifest_candidate(model, manifest_selected.get("model")),
}
selected = {name: values[0] for name, values in candidates.items() if values}
missing = [name for name in ("xplatform", "csv", "json", "lerobot", "rosbag", "model") if name not in selected]
shell_command = args.shell_command.strip()
if not shell_command and existing_manifest:
shell_command = manifest_config_text(existing_manifest, MANIFEST_SHELL_COMMAND_PATHS)
if not shell_command:
missing.append("shellCommand")
manifest = build_manifest(selected, shell_command) if not missing else None
status = "ready" if manifest is not None else "incomplete"
write_result = ""
if args.write_manifest:
if manifest is None:
write_result = "skipped: discovery result is incomplete"
else:
rendered = json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
if not write_remote_text(args.namespace, pod, args.manifest_path, rendered):
payload = {"status": "error", "message": f"failed to write manifest to {args.manifest_path}", "namespace": args.namespace}
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
return 2
write_result = "written"
payload = {
"status": status,
"namespace": args.namespace,
"apiPod": pod,
"workspaceMountPath": workspace_mount,
"roots": roots,
"excludedRoots": exclude_roots,
"acceptanceOutputIncluded": bool(args.include_acceptance_output),
"manifestPath": args.manifest_path,
"manifestSource": "existing" if existing_manifest else "",
"candidates": {name: [candidate_to_json(candidate) for candidate in values] for name, values in candidates.items()},
"selected": {name: candidate_to_json(candidate) for name, candidate in selected.items()},
"missing": missing,
"manifest": manifest,
"writeResult": write_result,
"nextActions": [],
}
if "shellCommand" in missing:
payload["nextActions"].append("Set POC_ACCEPTANCE_SHELL_COMMAND to the real validation command that should run inside the acceptance Job.")
if any(name != "shellCommand" for name in missing):
payload["nextActions"].append("Copy real fixture data onto the shared /workspace PVC, or set POC_ACCEPTANCE_DISCOVERY_ROOTS to the existing real-data roots.")
if status == "ready" and not args.write_manifest and not existing_manifest:
payload["nextActions"].append("Run this command again with --write-manifest, then run DEPLOY_ENV=poc npm run acceptance:poc-readiness.")
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
if args.write_manifest and manifest is None:
return 1
if args.require_complete and status != "ready":
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())