forked from longtao.wu/droboticflow
248 lines
12 KiB
Python
248 lines
12 KiB
Python
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "configure-poc-acceptance-fixtures.sh"
|
|
|
|
|
|
def write_fake_kubectl(tmp_path: pathlib.Path, calls_path: pathlib.Path, remote_root: pathlib.Path) -> pathlib.Path:
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
kubectl = fake_bin / "kubectl"
|
|
kubectl.write_text(
|
|
"#!/usr/bin/env python3\n"
|
|
"import pathlib\n"
|
|
"import shutil\n"
|
|
"import sys\n"
|
|
"import tarfile\n"
|
|
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
|
|
f"remote_root = pathlib.Path({str(remote_root)!r})\n"
|
|
"args = sys.argv[1:]\n"
|
|
"with calls_path.open('a', encoding='utf-8') as handle:\n"
|
|
" handle.write(' '.join(args) + '\\n')\n"
|
|
"def remote(path):\n"
|
|
" return remote_root / path.lstrip('/')\n"
|
|
"if args[:2] == ['get', 'namespace']:\n"
|
|
" raise SystemExit(0)\n"
|
|
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
|
|
" print('droboticflow-api-test')\n"
|
|
" raise SystemExit(0)\n"
|
|
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
|
|
" target = remote(args[-1])\n"
|
|
" script = args[args.index('-c') + 1]\n"
|
|
" if 'cat > \"$1\"' in script:\n"
|
|
" target.parent.mkdir(parents=True, exist_ok=True)\n"
|
|
" target.write_bytes(sys.stdin.buffer.read())\n"
|
|
" raise SystemExit(0 if target.is_file() and target.stat().st_size > 0 else 1)\n"
|
|
" if 'tar -xf -' in script:\n"
|
|
" if target.exists():\n"
|
|
" shutil.rmtree(target) if target.is_dir() else target.unlink()\n"
|
|
" target.mkdir(parents=True, exist_ok=True)\n"
|
|
" with tarfile.open(fileobj=sys.stdin.buffer, mode='r|') as archive:\n"
|
|
" archive.extractall(target)\n"
|
|
" raise SystemExit(0 if target.is_dir() else 1)\n"
|
|
" if 'test -f' in script:\n"
|
|
" raise SystemExit(0 if target.is_file() and target.stat().st_size > 0 else 1)\n"
|
|
" if 'test -d' in script:\n"
|
|
" raise SystemExit(0 if target.is_dir() else 1)\n"
|
|
" if 'test -e' in script:\n"
|
|
" raise SystemExit(0 if target.exists() else 1)\n"
|
|
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
|
|
"raise SystemExit(97)\n",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
return fake_bin
|
|
|
|
|
|
def make_fixture_tree(tmp_path: pathlib.Path) -> dict[str, pathlib.Path]:
|
|
source = tmp_path / "fixtures"
|
|
xplatform = source / "xplatform"
|
|
lerobot = source / "lerobot"
|
|
rosbag = source / "rosbag"
|
|
for path in (xplatform, lerobot, rosbag):
|
|
path.mkdir(parents=True)
|
|
(path / "marker.txt").write_text(path.name, encoding="utf-8")
|
|
csv_path = source / "records.csv"
|
|
csv_path.write_text("id,label,score\nclip-1,keep,0.9\n", encoding="utf-8")
|
|
json_path = source / "records.json"
|
|
json_path.write_text('{"items":[{"id":"clip-1","label":"keep","score":0.9}]}', encoding="utf-8")
|
|
training_json_path = source / "training-records.json"
|
|
training_json_path.write_text(
|
|
'{"items":[{"id":"sample-1","instruction":"push block","state":[0.1,0.2],"action":[0.3,0.4]}]}',
|
|
encoding="utf-8",
|
|
)
|
|
model_path = source / "model.json"
|
|
model_path.write_text('{"type":"constant","label":"accepted"}', encoding="utf-8")
|
|
yolo_onnx_path = source / "yolov8n.onnx"
|
|
yolo_onnx_path.write_bytes(b"real-onnx-placeholder")
|
|
return {
|
|
"xplatform": xplatform,
|
|
"csv": csv_path,
|
|
"json": json_path,
|
|
"training_json": training_json_path,
|
|
"lerobot": lerobot,
|
|
"rosbag": rosbag,
|
|
"model": model_path,
|
|
"yolo_onnx": yolo_onnx_path,
|
|
}
|
|
|
|
|
|
def run_script(tmp_path: pathlib.Path, extra_env: dict[str, str]) -> tuple[subprocess.CompletedProcess, pathlib.Path, pathlib.Path]:
|
|
calls_path = tmp_path / "kubectl-calls.log"
|
|
remote_root = tmp_path / "remote"
|
|
remote_root.mkdir()
|
|
fake_bin = write_fake_kubectl(tmp_path, calls_path, remote_root)
|
|
env = os.environ.copy()
|
|
for key in (
|
|
"DEPLOY_ENV",
|
|
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
|
|
"K8S_NAMESPACE",
|
|
"POC_ACCEPTANCE_AUDIT_RULES_JSON",
|
|
"POC_ACCEPTANCE_CSV_INPUT_PATH",
|
|
"POC_ACCEPTANCE_FIXTURE_DRY_RUN",
|
|
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
|
|
"POC_ACCEPTANCE_FIXTURE_TARGET_DIR",
|
|
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON",
|
|
"POC_ACCEPTANCE_JSON_INPUT_PATH",
|
|
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_CSV_INPUT_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_JSON_INPUT_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_LEROBOT_SOURCE_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_MODEL_JSON_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_ROSBAG_SOURCE_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_TRAINING_JSON_INPUT_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_XPLATFORM_SOURCE_PATH",
|
|
"POC_ACCEPTANCE_LOCAL_YOLO_ONNX_MODEL_PATH",
|
|
"POC_ACCEPTANCE_MODEL_JSON_PATH",
|
|
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
|
|
"POC_ACCEPTANCE_SHELL_COMMAND",
|
|
"POC_ACCEPTANCE_TRAINING_JSON_INPUT_PATH",
|
|
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
|
|
"POC_ACCEPTANCE_YOLO_ONNX_MODEL_PATH",
|
|
"WORKSPACE_MOUNT_PATH",
|
|
):
|
|
env.pop(key, None)
|
|
env.update(
|
|
{
|
|
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
|
|
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"WORKSPACE_MOUNT_PATH": "/workspace",
|
|
}
|
|
)
|
|
env.update(extra_env)
|
|
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
return result, calls_path, remote_root
|
|
|
|
|
|
def fixture_env(paths: dict[str, pathlib.Path]) -> dict[str, str]:
|
|
return {
|
|
"POC_ACCEPTANCE_LOCAL_XPLATFORM_SOURCE_PATH": str(paths["xplatform"]),
|
|
"POC_ACCEPTANCE_LOCAL_CSV_INPUT_PATH": str(paths["csv"]),
|
|
"POC_ACCEPTANCE_LOCAL_JSON_INPUT_PATH": str(paths["json"]),
|
|
"POC_ACCEPTANCE_LOCAL_LEROBOT_SOURCE_PATH": str(paths["lerobot"]),
|
|
"POC_ACCEPTANCE_LOCAL_ROSBAG_SOURCE_PATH": str(paths["rosbag"]),
|
|
"POC_ACCEPTANCE_LOCAL_MODEL_JSON_PATH": str(paths["model"]),
|
|
"POC_ACCEPTANCE_SHELL_COMMAND": "python3 /workspace/datasets/fixtures/real-fixtures/check.py",
|
|
}
|
|
|
|
|
|
def test_configure_poc_acceptance_fixtures_requires_shell_command_before_kubectl(tmp_path):
|
|
paths = make_fixture_tree(tmp_path)
|
|
env = fixture_env(paths)
|
|
env.pop("POC_ACCEPTANCE_SHELL_COMMAND")
|
|
|
|
result, calls_path, _remote_root = run_script(tmp_path, env)
|
|
|
|
assert result.returncode == 1
|
|
assert "POC_ACCEPTANCE_SHELL_COMMAND is required" in result.stderr
|
|
assert not calls_path.exists()
|
|
|
|
|
|
def test_configure_poc_acceptance_fixtures_dry_run_does_not_touch_kubectl(tmp_path):
|
|
paths = make_fixture_tree(tmp_path)
|
|
env = fixture_env(paths)
|
|
env["POC_ACCEPTANCE_FIXTURE_DRY_RUN"] = "true"
|
|
|
|
result, calls_path, _remote_root = run_script(tmp_path, env)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "POC acceptance fixture dry-run" in result.stdout
|
|
assert "manifest=/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" in result.stdout
|
|
assert "shellCommandConfigured=true" in result.stdout
|
|
assert not calls_path.exists()
|
|
|
|
|
|
def test_configure_poc_acceptance_fixtures_copies_real_inputs_and_writes_manifest(tmp_path):
|
|
paths = make_fixture_tree(tmp_path)
|
|
env = fixture_env(paths)
|
|
env.update(
|
|
{
|
|
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON": '{"status":"decision"}',
|
|
"POC_ACCEPTANCE_AUDIT_RULES_JSON": '[{"id":"all","field":"id","operator":"exists","samplePercent":100}]',
|
|
}
|
|
)
|
|
|
|
result, calls_path, remote_root = run_script(tmp_path, env)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "POC acceptance fixtures applied: namespace=droboticflow manifest=/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" in result.stdout
|
|
assert 'export POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH="/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"' in result.stdout
|
|
manifest_path = remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "poc-operator-acceptance.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
assert manifest["fixtures"] == {
|
|
"xplatform": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
|
|
"csv": "/workspace/datasets/fixtures/poc-operator-acceptance/csv/records.csv",
|
|
"json": "/workspace/datasets/fixtures/poc-operator-acceptance/json/records.json",
|
|
"lerobot": "/workspace/datasets/fixtures/poc-operator-acceptance/lerobot",
|
|
"rosbag": "/workspace/datasets/fixtures/poc-operator-acceptance/rosbag",
|
|
"model": "/workspace/datasets/fixtures/poc-operator-acceptance/model/model.json",
|
|
}
|
|
assert manifest["shellCommand"] == "python3 /workspace/datasets/fixtures/real-fixtures/check.py"
|
|
assert manifest["json"]["fieldMap"] == {"status": "decision"}
|
|
assert manifest["audit"]["rules"][0]["samplePercent"] == 100
|
|
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "xplatform" / "marker.txt").read_text(encoding="utf-8") == "xplatform"
|
|
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "csv" / "records.csv").is_file()
|
|
assert "get namespace droboticflow" in calls_path.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_configure_poc_acceptance_fixtures_writes_training_and_yolo_model_paths(tmp_path):
|
|
paths = make_fixture_tree(tmp_path)
|
|
env = {
|
|
**fixture_env(paths),
|
|
"POC_ACCEPTANCE_LOCAL_TRAINING_JSON_INPUT_PATH": str(paths["training_json"]),
|
|
"POC_ACCEPTANCE_LOCAL_YOLO_ONNX_MODEL_PATH": str(paths["yolo_onnx"]),
|
|
}
|
|
|
|
result, _calls_path, remote_root = run_script(tmp_path, env)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
manifest_path = remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "poc-operator-acceptance.json"
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
assert manifest["fixtures"]["training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
|
|
assert manifest["fixtures"]["policy_training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
|
|
assert manifest["fixtures"]["vla_training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
|
|
assert manifest["fixtures"]["openvla_training_json"] == "/workspace/datasets/fixtures/poc-operator-acceptance/json/training-records.json"
|
|
assert manifest["fixtures"]["yolo_onnx_model"] == "/workspace/datasets/fixtures/poc-operator-acceptance/yolo/models/yolov8n.onnx"
|
|
assert manifest["fixtures"]["dnn_model_path"] == "/workspace/datasets/fixtures/poc-operator-acceptance/yolo/models/yolov8n.onnx"
|
|
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "json" / "training-records.json").is_file()
|
|
assert (remote_root / "workspace" / "datasets" / "fixtures" / "poc-operator-acceptance" / "yolo" / "models" / "yolov8n.onnx").is_file()
|
|
|
|
|
|
def test_configure_poc_acceptance_fixtures_rejects_local_file_names_that_escape_workspace(tmp_path):
|
|
paths = make_fixture_tree(tmp_path)
|
|
spaced_csv = paths["csv"].with_name("records with spaces.csv")
|
|
paths["csv"].rename(spaced_csv)
|
|
paths["csv"] = spaced_csv
|
|
env = fixture_env(paths)
|
|
|
|
result, calls_path, _remote_root = run_script(tmp_path, env)
|
|
|
|
assert result.returncode == 1
|
|
assert "POC_ACCEPTANCE_CSV_INPUT_PATH must be an absolute workspace path without spaces" in result.stderr
|
|
assert not calls_path.exists()
|