droboticflow/tests/backend/configure_poc_acceptance_fixtures_script_test.py
2026-07-07 18:57:49 +08:00

248 lines
12 KiB
Python

import json
import os
import pathlib
import subprocess
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "configure-poc-acceptance-fixtures.sh"
def write_fake_kubectl(tmp_path: pathlib.Path, calls_path: pathlib.Path, remote_root: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import pathlib\n"
"import shutil\n"
"import sys\n"
"import tarfile\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
f"remote_root = pathlib.Path({str(remote_root)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"def remote(path):\n"
" return remote_root / path.lstrip('/')\n"
"if args[:2] == ['get', 'namespace']:\n"
" raise SystemExit(0)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" target = remote(args[-1])\n"
" script = args[args.index('-c') + 1]\n"
" if 'cat > \"$1\"' in script:\n"
" target.parent.mkdir(parents=True, exist_ok=True)\n"
" target.write_bytes(sys.stdin.buffer.read())\n"
" raise SystemExit(0 if target.is_file() and target.stat().st_size > 0 else 1)\n"
" if 'tar -xf -' in script:\n"
" if target.exists():\n"
" shutil.rmtree(target) if target.is_dir() else target.unlink()\n"
" target.mkdir(parents=True, exist_ok=True)\n"
" with tarfile.open(fileobj=sys.stdin.buffer, mode='r|') as archive:\n"
" archive.extractall(target)\n"
" raise SystemExit(0 if target.is_dir() else 1)\n"
" if 'test -f' in script:\n"
" raise SystemExit(0 if target.is_file() and target.stat().st_size > 0 else 1)\n"
" if 'test -d' in script:\n"
" raise SystemExit(0 if target.is_dir() else 1)\n"
" if 'test -e' in script:\n"
" raise SystemExit(0 if target.exists() else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def make_fixture_tree(tmp_path: pathlib.Path) -> dict[str, pathlib.Path]:
source = tmp_path / "fixtures"
xplatform = source / "xplatform"
lerobot = source / "lerobot"
rosbag = source / "rosbag"
for path in (xplatform, lerobot, rosbag):
path.mkdir(parents=True)
(path / "marker.txt").write_text(path.name, encoding="utf-8")
csv_path = source / "records.csv"
csv_path.write_text("id,label,score\nclip-1,keep,0.9\n", encoding="utf-8")
json_path = source / "records.json"
json_path.write_text('{"items":[{"id":"clip-1","label":"keep","score":0.9}]}', encoding="utf-8")
training_json_path = source / "training-records.json"
training_json_path.write_text(
'{"items":[{"id":"sample-1","instruction":"push block","state":[0.1,0.2],"action":[0.3,0.4]}]}',
encoding="utf-8",
)
model_path = source / "model.json"
model_path.write_text('{"type":"constant","label":"accepted"}', encoding="utf-8")
yolo_onnx_path = source / "yolov8n.onnx"
yolo_onnx_path.write_bytes(b"real-onnx-placeholder")
return {
"xplatform": xplatform,
"csv": csv_path,
"json": json_path,
"training_json": training_json_path,
"lerobot": lerobot,
"rosbag": rosbag,
"model": model_path,
"yolo_onnx": yolo_onnx_path,
}
def run_script(tmp_path: pathlib.Path, extra_env: dict[str, str]) -> tuple[subprocess.CompletedProcess, pathlib.Path, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
remote_root = tmp_path / "remote"
remote_root.mkdir()
fake_bin = write_fake_kubectl(tmp_path, calls_path, remote_root)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_NAMESPACE",
"POC_ACCEPTANCE_AUDIT_RULES_JSON",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_FIXTURE_DRY_RUN",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_FIXTURE_TARGET_DIR",
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_LOCAL_CSV_INPUT_PATH",
"POC_ACCEPTANCE_LOCAL_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LOCAL_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_LOCAL_MODEL_JSON_PATH",
"POC_ACCEPTANCE_LOCAL_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_LOCAL_TRAINING_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LOCAL_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_LOCAL_YOLO_ONNX_MODEL_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_ACCEPTANCE_TRAINING_JSON_INPUT_PATH",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_YOLO_ONNX_MODEL_PATH",
"WORKSPACE_MOUNT_PATH",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"WORKSPACE_MOUNT_PATH": "/workspace",
}
)
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path, remote_root
def fixture_env(paths: dict[str, pathlib.Path]) -> dict[str, str]:
return {
"POC_ACCEPTANCE_LOCAL_XPLATFORM_SOURCE_PATH": str(paths["xplatform"]),
"POC_ACCEPTANCE_LOCAL_CSV_INPUT_PATH": str(paths["csv"]),
"POC_ACCEPTANCE_LOCAL_JSON_INPUT_PATH": str(paths["json"]),
"POC_ACCEPTANCE_LOCAL_LEROBOT_SOURCE_PATH": str(paths["lerobot"]),
"POC_ACCEPTANCE_LOCAL_ROSBAG_SOURCE_PATH": str(paths["rosbag"]),
"POC_ACCEPTANCE_LOCAL_MODEL_JSON_PATH": str(paths["model"]),
"POC_ACCEPTANCE_SHELL_COMMAND": "python3 /workspace/datasets/fixtures/real-fixtures/check.py",
}
def test_configure_poc_acceptance_fixtures_requires_shell_command_before_kubectl(tmp_path):
paths = make_fixture_tree(tmp_path)
env = fixture_env(paths)
env.pop("POC_ACCEPTANCE_SHELL_COMMAND")
result, calls_path, _remote_root = run_script(tmp_path, env)
assert result.returncode == 1
assert "POC_ACCEPTANCE_SHELL_COMMAND is required" in result.stderr
assert not calls_path.exists()
def test_configure_poc_acceptance_fixtures_dry_run_does_not_touch_kubectl(tmp_path):
paths = make_fixture_tree(tmp_path)
env = fixture_env(paths)
env["POC_ACCEPTANCE_FIXTURE_DRY_RUN"] = "true"
result, calls_path, _remote_root = run_script(tmp_path, env)
assert result.returncode == 0, result.stderr + result.stdout
assert "POC acceptance fixture dry-run" in result.stdout
assert "manifest=/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" in result.stdout
assert "shellCommandConfigured=true" in result.stdout
assert not calls_path.exists()
def test_configure_poc_acceptance_fixtures_copies_real_inputs_and_writes_manifest(tmp_path):
paths = make_fixture_tree(tmp_path)
env = fixture_env(paths)
env.update(
{
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON": '{"status":"decision"}',
"POC_ACCEPTANCE_AUDIT_RULES_JSON": '[{"id":"all","field":"id","operator":"exists","samplePercent":100}]',
}
)
result, calls_path, remote_root = run_script(tmp_path, env)
assert result.returncode == 0, result.stderr + result.stdout
assert "POC acceptance fixtures applied: namespace=droboticflow manifest=/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" in result.stdout
assert 'export POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH="/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"' in result.stdout
manifest_path = remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "poc-operator-acceptance.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
assert manifest["fixtures"] == {
"xplatform": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
"csv": "/workspace/datasets/fixtures/poc-operator-acceptance/csv/records.csv",
"json": "/workspace/datasets/fixtures/poc-operator-acceptance/json/records.json",
"lerobot": "/workspace/datasets/fixtures/poc-operator-acceptance/lerobot",
"rosbag": "/workspace/datasets/fixtures/poc-operator-acceptance/rosbag",
"model": "/workspace/datasets/fixtures/poc-operator-acceptance/model/model.json",
}
assert manifest["shellCommand"] == "python3 /workspace/datasets/fixtures/real-fixtures/check.py"
assert manifest["json"]["fieldMap"] == {"status": "decision"}
assert manifest["audit"]["rules"][0]["samplePercent"] == 100
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "xplatform" / "marker.txt").read_text(encoding="utf-8") == "xplatform"
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "csv" / "records.csv").is_file()
assert "get namespace droboticflow" in calls_path.read_text(encoding="utf-8")
def test_configure_poc_acceptance_fixtures_writes_training_and_yolo_model_paths(tmp_path):
paths = make_fixture_tree(tmp_path)
env = {
**fixture_env(paths),
"POC_ACCEPTANCE_LOCAL_TRAINING_JSON_INPUT_PATH": str(paths["training_json"]),
"POC_ACCEPTANCE_LOCAL_YOLO_ONNX_MODEL_PATH": str(paths["yolo_onnx"]),
}
result, _calls_path, remote_root = run_script(tmp_path, env)
assert result.returncode == 0, result.stderr + result.stdout
manifest_path = remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "poc-operator-acceptance.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
assert manifest["fixtures"]["training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
assert manifest["fixtures"]["policy_training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
assert manifest["fixtures"]["vla_training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
assert manifest["fixtures"]["openvla_training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
assert manifest["fixtures"]["yolo_onnx_model"] == "/workspace/datasets/fixtures/poc-operator-acceptance/yolo/models/yolov8n.onnx"
assert manifest["fixtures"]["dnn_model_path"] == "/workspace/datasets/fixtures/poc-operator-acceptance/yolo/models/yolov8n.onnx"
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "json" / "training-records.json").is_file()
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "yolo" / "models" / "yolov8n.onnx").is_file()
def test_configure_poc_acceptance_fixtures_rejects_local_file_names_that_escape_workspace(tmp_path):
paths = make_fixture_tree(tmp_path)
spaced_csv = paths["csv"].with_name("records with spaces.csv")
paths["csv"].rename(spaced_csv)
paths["csv"] = spaced_csv
env = fixture_env(paths)
result, calls_path, _remote_root = run_script(tmp_path, env)
assert result.returncode == 1
assert "POC_ACCEPTANCE_CSV_INPUT_PATH must be an absolute workspace path without spaces" in result.stderr
assert not calls_path.exists()