forked from longtao.wu/droboticflow
253 lines
14 KiB
Python
253 lines
14 KiB
Python
"""Public CLI and lifecycle refusal tests using only synthetic local files."""
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from scripts.internal_test_lane.lifecycle import Lane, LaneError
|
|
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
CLI = REPO / "scripts" / "internal_test_lane.py"
|
|
|
|
|
|
def synthetic_bundle(directory: Path) -> Path:
|
|
samples = []
|
|
assets = []
|
|
for project in ("BLF-data", "SL-data"):
|
|
for number in range(3):
|
|
key = f"{project.lower()}-{number}"
|
|
media = {}
|
|
for field in ("leftVideo", "rightVideo", "leftPTS", "rightPTS", "imu", "sourceManifest"):
|
|
rel = f"templates/{key}/{field}.bin"
|
|
payload = f"synthetic:{key}:{field}".encode()
|
|
path = directory / rel
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(payload)
|
|
media[field] = rel
|
|
assets.append({"path": rel, "sha256": hashlib.sha256(payload).hexdigest(), "size": len(payload)})
|
|
mode = "scratch" if number == 0 else "snapshot"
|
|
sample = {"key": key, "project": project, "mode": mode, "source": {"captureId": f"synthetic-{key}", "version": "v1"}, "media": media, "durationSeconds": 4}
|
|
if mode == "snapshot":
|
|
sample["snapshot"] = {"stage": "annotation", "slices": [{"start": 0, "end": 4, "name": "test"}], "actions": [{"start": 0, "end": 4, "description": "synthetic", "isInvalid": False}]}
|
|
samples.append(sample)
|
|
dictionaries = {name: {"sliceNames": ["test"], "descriptions": ["synthetic"], "discardReasons": ["bad"]} for name in ("BLF-data", "SL-data")}
|
|
bundle = directory / "bundle.json"
|
|
bundle.write_text(json.dumps({"schemaVersion": "droboticflow.internal-test-seed/v1", "dictionaries": dictionaries, "samples": samples, "assets": assets}), encoding="utf-8")
|
|
return bundle
|
|
|
|
|
|
class LifecycleTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.base = Path(self.temp.name).resolve()
|
|
self.bundle = synthetic_bundle(self.base / "source")
|
|
self.lane = Lane(self.base / "lane")
|
|
|
|
def test_import_is_immutable_and_rejects_hash_change(self):
|
|
first = self.lane.import_bundle(self.bundle)
|
|
self.assertEqual(first["samples"], 6)
|
|
self.assertFalse(first["reused"])
|
|
self.assertTrue(self.lane.import_bundle(self.bundle)["reused"])
|
|
asset = self.base / "source" / "templates" / "blf-data-0" / "imu.bin"
|
|
asset.write_bytes(b"changed")
|
|
with self.assertRaisesRegex(LaneError, "hash/size mismatch"):
|
|
self.lane.import_bundle(self.bundle)
|
|
imported_asset = self.lane.root / "imports" / first["importId"] / "templates" / "blf-data-0" / "imu.bin"
|
|
self.assertNotEqual(imported_asset.read_bytes(), b"changed")
|
|
|
|
def test_import_refuses_escape_and_symlink(self):
|
|
value = json.loads(self.bundle.read_text())
|
|
value["samples"][0]["media"]["leftVideo"] = "../outside"
|
|
self.bundle.write_text(json.dumps(value))
|
|
with self.assertRaisesRegex(LaneError, "unsafe asset path"):
|
|
self.lane.import_bundle(self.bundle)
|
|
|
|
self.bundle = synthetic_bundle(self.base / "source")
|
|
path = self.base / "source" / "templates" / "blf-data-0" / "leftVideo.bin"
|
|
path.unlink()
|
|
path.symlink_to(self.base / "source" / "templates" / "blf-data-0" / "rightVideo.bin")
|
|
with self.assertRaisesRegex(LaneError, "symlink"):
|
|
self.lane.import_bundle(self.bundle)
|
|
|
|
def test_cli_status_reports_local_preparation_only(self):
|
|
self.lane.import_bundle(self.bundle)
|
|
completed = subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "status"], cwd=REPO, capture_output=True, text=True, check=False)
|
|
self.assertEqual(completed.returncode, 0, completed.stderr)
|
|
status = json.loads(completed.stdout)
|
|
self.assertEqual(status["remoteState"], "not-verified-by-this-tool")
|
|
self.assertEqual(len(status["imports"]), 1)
|
|
|
|
def test_cli_credentials_rejects_symlink_ancestor_inside_owned_root(self):
|
|
from test_environment import config
|
|
|
|
self.lane.import_bundle(self.bundle)
|
|
config_path = self.base / "config.json"
|
|
config_path.write_text(json.dumps(config()))
|
|
inside = self.lane.root / "inside"
|
|
inside.mkdir()
|
|
(self.lane.root / "link").symlink_to(inside, target_is_directory=True)
|
|
output = self.lane.root / "link" / "credentials"
|
|
completed = subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "credentials", "--config", str(config_path), "--out", str(output)], cwd=REPO, capture_output=True, text=True, check=False)
|
|
self.assertEqual(completed.returncode, 2)
|
|
self.assertIn("symlink", completed.stderr)
|
|
self.assertFalse((inside / "credentials").exists())
|
|
|
|
def test_prepare_rejects_modified_import_manifest_or_inventory(self):
|
|
from test_environment import config
|
|
|
|
imported = self.lane.import_bundle(self.bundle)
|
|
config_path = self.base / "config.json"
|
|
config_path.write_text(json.dumps(config()))
|
|
source = self.lane.root / "imports" / imported["importId"]
|
|
manifest_path = source / "manifest.json"
|
|
original_manifest = manifest_path.read_text()
|
|
manifest = json.loads(original_manifest)
|
|
manifest["samples"][0]["source"]["version"] = "tampered"
|
|
manifest_path.write_text(json.dumps(manifest))
|
|
with self.assertRaisesRegex(LaneError, "digest no longer matches"):
|
|
self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO)
|
|
manifest_path.write_text(original_manifest)
|
|
inventory_path = source / "inventory.json"
|
|
inventory = json.loads(inventory_path.read_text())
|
|
first = next(iter(inventory))
|
|
inventory[first]["sha256"] = "0" * 64
|
|
inventory_path.write_text(json.dumps(inventory))
|
|
with self.assertRaisesRegex(LaneError, "digest no longer matches"):
|
|
self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO)
|
|
|
|
def test_cleanup_retains_latest_three_and_checks_preview(self):
|
|
self.lane.import_bundle(self.bundle)
|
|
batches = self.lane.root / "batches"
|
|
batches.mkdir()
|
|
for number in range(4):
|
|
batch = batches / f"test-{number}"
|
|
batch.mkdir()
|
|
(batch / "batch.json").write_text(json.dumps({"batchId": batch.name, "state": "archived", "archivedAt": f"2026-09-0{number + 1}T00:00:00Z"}))
|
|
(batch / "result.txt").write_text(f"synthetic-{number}")
|
|
with self.assertRaisesRegex(LaneError, "latest three"):
|
|
self.lane.cleanup_plan("test-3")
|
|
plan = self.lane.cleanup_plan("test-0")
|
|
self.assertEqual(plan["clusterCleanup"], "manual-not-performed")
|
|
with self.assertRaisesRegex(LaneError, "confirmation"):
|
|
self.lane.clean("test-0", "0" * 64)
|
|
(batches / "test-0" / "result.txt").write_text("changed")
|
|
with self.assertRaisesRegex(LaneError, "changed after cleanup"):
|
|
self.lane.clean("test-0", plan["confirmSha256"])
|
|
plan = self.lane.cleanup_plan("test-0")
|
|
result = self.lane.clean("test-0", plan["confirmSha256"])
|
|
self.assertTrue(result["removed"])
|
|
self.assertFalse((batches / "test-0").exists())
|
|
self.assertTrue((self.lane.root / "imports").exists())
|
|
|
|
def test_prepare_is_new_batch_only_and_never_infers_remote_state(self):
|
|
from tests.internal_test_lane.test_environment import config
|
|
|
|
imported = self.lane.import_bundle(self.bundle)
|
|
candidate = config()
|
|
config_path = self.base / "config.json"
|
|
config_path.write_text(json.dumps(candidate))
|
|
|
|
def fake_run(args, **kwargs):
|
|
Path(args[args.index("--out") + 1]).write_text('{"synthetic":true}')
|
|
return subprocess.CompletedProcess(args, 0, "", "")
|
|
|
|
with mock.patch("scripts.internal_test_lane.lifecycle.subprocess.run", side_effect=fake_run) as call:
|
|
record = self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO)
|
|
self.assertEqual(record["state"], "prepared")
|
|
self.assertEqual(record["remoteState"], "unknown")
|
|
package = self.lane.root / "batches" / "acceptance-1"
|
|
initializer = json.loads((package / "initializer.json").read_text())
|
|
self.assertEqual(initializer["kind"], "Pod")
|
|
self.assertEqual(initializer["metadata"]["namespace"], "droboticflow-internal-test")
|
|
self.assertEqual(record["initializerSha256"], hashlib.sha256((package / "initializer.json").read_bytes()).hexdigest())
|
|
self.assertEqual(call.call_args.kwargs["env"]["GOPROXY"], "off")
|
|
with self.assertRaisesRegex(LaneError, "already exists"):
|
|
self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO)
|
|
|
|
def test_cli_records_seed_and_http_evidence_without_remote_claim(self):
|
|
from test_environment import config
|
|
|
|
imported = self.lane.import_bundle(self.bundle)
|
|
cfg = config()
|
|
config_path = self.base / "config.json"
|
|
config_path.write_text(json.dumps(cfg))
|
|
source = self.lane.root / "imports" / imported["importId"]
|
|
manifest = json.loads((source / "manifest.json").read_text())
|
|
inventory = json.loads((source / "inventory.json").read_text())
|
|
plan = {
|
|
"batchId": "acceptance-1", "database": "droboticflow_internal_test_acceptance_1",
|
|
"assetsRoot": str(source), "workspaceRoot": str(self.base / "workspace"),
|
|
"manifestSha256": "f" * 64, "sha256": "a" * 64,
|
|
"assets": [{"path": path, **entry} for path, entry in sorted(inventory.items())],
|
|
"samples": [{"key": sample["key"], "project": sample["project"], "mode": sample["mode"],
|
|
"batchId": "internal-test-batch-" + sample["key"],
|
|
"taskId": "internal-test-task-" + sample["key"], "itemId": "internal-test-item-" + sample["key"]}
|
|
for sample in manifest["samples"]],
|
|
}
|
|
|
|
def fake_run(args, **kwargs):
|
|
Path(args[args.index("--out") + 1]).write_text(json.dumps(plan))
|
|
return subprocess.CompletedProcess(args, 0, "", "")
|
|
|
|
with mock.patch("scripts.internal_test_lane.lifecycle.subprocess.run", side_effect=fake_run):
|
|
prepared = self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO)
|
|
plan_path = self.lane.root / "batches" / "acceptance-1" / "seed-plan.json"
|
|
target = {"kind": "local", "databaseName": plan["database"], "hostname": "127.0.0.1", "namespace": "local"}
|
|
|
|
def record(stage, result, plan_sha="a" * 64):
|
|
result_path = self.base / f"{stage}-result.json"
|
|
result_path.write_text(json.dumps(result))
|
|
metadata = {"schemaVersion": 1, "batchId": "acceptance-1", "stage": stage,
|
|
"reviewer": "internal tester", "observedAt": "2026-09-27T12:00:00Z", "target": target,
|
|
"buildCommit": cfg["buildCommit"], "image": cfg["image"],
|
|
"configSha256": prepared["configSha256"],
|
|
"approvedExecutionPlanSha256": plan_sha,
|
|
"resultSha256": hashlib.sha256(result_path.read_bytes()).hexdigest()}
|
|
metadata_path = self.base / f"{stage}-evidence.json"
|
|
metadata_path.write_text(json.dumps(metadata))
|
|
return subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "record-evidence",
|
|
"--batch", "acceptance-1", "--evidence", str(metadata_path),
|
|
"--plan", str(plan_path), "--result", str(result_path)],
|
|
cwd=REPO, capture_output=True, text=True, check=False)
|
|
|
|
seed_result = {"database": plan["database"], "planSha256": plan["sha256"], "created": True,
|
|
"projects": 2, "samples": 6, "accounts": 5}
|
|
refused = record("seeded", seed_result, plan_sha="b" * 64)
|
|
self.assertEqual(refused.returncode, 2)
|
|
self.assertIn("hash", refused.stderr)
|
|
sample = plan["samples"][0]
|
|
http_result = {"batchId": "acceptance-1", "sampleKey": sample["key"], "taskId": sample["taskId"],
|
|
"itemId": sample["itemId"], "acceptanceItemId": "human-item-fanyu-v5-prepackage-acceptance-" + sample["batchId"],
|
|
"status": "accepted", "checks": [{"name": "full-http-workflow", "status": "passed"}]}
|
|
before_seed = record("business-verified", http_result)
|
|
self.assertEqual(before_seed.returncode, 2)
|
|
self.assertIn("requires batch state seeded", before_seed.stderr)
|
|
self.assertEqual(record("seeded", seed_result).returncode, 0)
|
|
self.assertEqual(record("business-verified", http_result).returncode, 0)
|
|
completed = subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "status"], cwd=REPO, capture_output=True, text=True, check=False)
|
|
self.assertEqual(completed.returncode, 0)
|
|
row = json.loads(completed.stdout)["batches"][0]
|
|
self.assertEqual(row["state"], "business-verified")
|
|
self.assertEqual(row["remoteState"], "not-verified-by-this-tool")
|
|
self.assertEqual(row["evidence"]["business-verified"]["scope"], "single-sample-http-workflow")
|
|
self.assertEqual(row["businessAcceptance"], "operator-report-imported")
|
|
archive_path = self.base / "archive.json"
|
|
archive_path.write_text(json.dumps({"batchId": "acceptance-1", "tasksComplete": True,
|
|
"environmentStopped": True, "detached": True, "reviewer": "internal tester"}))
|
|
archived = subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "archive",
|
|
"--batch", "acceptance-1", "--evidence", str(archive_path)],
|
|
cwd=REPO, capture_output=True, text=True, check=False)
|
|
self.assertEqual(archived.returncode, 0, archived.stderr)
|
|
self.assertIn("business-verified", json.loads(archived.stdout)["evidence"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|