forked from longtao.wu/droboticflow
Add scripts/part_index_migration.py, which produces the missing part.json for a part that was delivered in the directory form. It reuses the package operator's own builder by loading the operator source the way the runner does, so a migrated index is structurally identical to an online package and byte-identical to what the same evidence would have produced, apart from a lineage.migratedFrom marker. A test compares both routes on one evidence set to keep the two from forking. The tool never writes object storage: publication has to go through the API/Worker path so the owner marker, conditional create and no-overwrite guarantees stay in effect. It refuses to run when the delivery directory lacks a completion manifest, when a slice has an unknown disposition, when the revision has no per-slice disposition evidence, and when the part already has a part.json with different bytes. Also fix a leak found while building this: the index wrote the operator's absolute workspace path into calibration[].path. It now records the logical delivery-relative path, matching the directory form's packages-manifest, and keeps the local location out of customer-facing output. Verification: 19 operator and migration tests pass. tests/backend failure set is identical to the HEAD baseline (25 failed, 829 subtests passed, 516 vs 525 passing), so no regression.
185 lines
11 KiB
Python
185 lines
11 KiB
Python
"""The delivered-directory migration must reuse the operator's own builder.
|
|
|
|
These tests pin the property that matters for a one-way migration: the index a
|
|
migration tool produces from an already-delivered directory is byte-identical to
|
|
the index the package operator would have produced for the same revision. That
|
|
only holds while both share one implementation, so this test is also the guard
|
|
against someone forking the builder.
|
|
"""
|
|
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from backend.operators.builtin.package import operator as package
|
|
|
|
from tests.backend.test_part_index_package import PART_NODE, helpers
|
|
from tests.backend.test_slice_discard_package import fixture
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def migration_module():
|
|
path = ROOT / "scripts" / "part_index_migration.py"
|
|
spec = importlib.util.spec_from_file_location("part_index_migration_under_test", path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def delivered_part(tmp_path, dispositions):
|
|
"""Build a part directory shaped like a published v6 delivery, plus its
|
|
materialization revision, exactly as the operator and worker leave them."""
|
|
delivery_root = tmp_path / "part0001"
|
|
delivery_root.mkdir(parents=True)
|
|
(delivery_root / "calibration").mkdir()
|
|
for index, disposition in enumerate(dispositions, start=1):
|
|
slice_id = f"slice-{index:06d}"
|
|
folder = delivery_root / "clips" / slice_id
|
|
folder.mkdir(parents=True)
|
|
actions = [] if disposition == "discard" else [{"id": f"a{index}", "description": "拿起物品", "isInvalid": False}]
|
|
(folder / "annotation.json").write_text(json.dumps({"schemaVersion": "droboticflow.timestamped-description/v3", "sliceId": slice_id, "actions": actions, "descriptions": actions}), encoding="utf8")
|
|
(folder / "synchronized-slice.json").write_text(json.dumps({
|
|
"schemaVersion": "droboticflow.synchronized-slice/v1",
|
|
"deliveryDisposition": disposition,
|
|
"boundaryContract": "hardware-frame-v1",
|
|
"maxStereoAlignmentErrorUs": 120,
|
|
"boundaryTimestampUs": [1000000, 2000000],
|
|
}), encoding="utf8")
|
|
(folder / "left.mp4").write_bytes(f"{slice_id}-left".encode())
|
|
(folder / "right.mp4").write_bytes(f"{slice_id}-right".encode())
|
|
(folder / "left_pts.csv").write_text("frame_index,timestamp_ms,clip_time_ms\n0,1000,0\n", encoding="utf8")
|
|
(folder / "right_pts.csv").write_text("frame_index,timestamp_ms,clip_time_ms\n0,1000,0\n", encoding="utf8")
|
|
(folder / "imu.csv").write_text("timestamp_ms,clip_time_ms,x,y,z,type\n1000,0,0,0,0,accel\n", encoding="utf8")
|
|
|
|
# Mirrors the runtime: these evidence files are projections of one accepted
|
|
# summary, so the tool and the online path read the same facts.
|
|
(delivery_root / "acceptance-record.json").write_text(json.dumps({"schemaVersion": "droboticflow.capture-acceptance/v1", "finalAcceptance": {
|
|
"confirmed": True, "sampled": True, "samplingDecision": "selected", "processingProfile": "raw_human",
|
|
"poseDelivery": {"headpose": "exclude", "handpose": "exclude"},
|
|
}}), encoding="utf8")
|
|
(delivery_root / "capture-manifest.json").write_text(json.dumps({"schemaVersion": "droboticflow.capture-delivery/v1", "capture": {"captureId": "CAP-1", "prefix": "raw/part0001"}}), encoding="utf8")
|
|
(delivery_root / "quality-review.json").write_text(json.dumps({"schemaVersion": "droboticflow.capture-quality-review/v1", "reviews": [{"taskId": "task-1", "revision": 1, "decision": "approved", "reviewerUserId": "user-qc", "createdAt": "2026-01-01T00:00:00Z", "comment": "INTERNAL-ONLY"}]}), encoding="utf8")
|
|
|
|
# The accepted materialization revision the delivery was built from.
|
|
inputs = fixture(tmp_path, dispositions)
|
|
revision_root = Path(inputs[0]["path"])
|
|
rows = json.loads((revision_root / "revision-manifest.json").read_text(encoding="utf8"))["items"]
|
|
for row in rows:
|
|
slice_id = row["sliceId"]
|
|
source = revision_root / "slices" / slice_id
|
|
target = delivery_root / "clips" / slice_id
|
|
# The delivered copy is authoritative for the index; keep them identical
|
|
# so any drift would show up as a byte difference in the test.
|
|
for name in ("annotation.json", "synchronized-slice.json"):
|
|
target.joinpath(name).write_bytes(source.joinpath(name).read_bytes())
|
|
(delivery_root / "packages-manifest.json").write_text(json.dumps({"schemaVersion": "droboticflow.delivery.manifest/v6", "capture": {"captureId": "CAP-1"}, "lineage": {"acceptedSliceDirectoryArtifactId": "accepted"}}), encoding="utf8")
|
|
# Calibration is the one input a delivered directory must carry itself, so
|
|
# copy the very bytes the online package would reference. Both routes then
|
|
# see identical inputs and any byte difference is a real contract break.
|
|
calibrations = Path(inputs[0]["path"]).parent / "package-test-calibrations"
|
|
for source in sorted(calibrations.iterdir()):
|
|
(delivery_root / "calibration" / source.name).write_bytes(source.read_bytes())
|
|
refresh_completion_manifest(delivery_root)
|
|
return delivery_root, revision_root, inputs
|
|
|
|
|
|
def refresh_completion_manifest(delivery_root: Path) -> None:
|
|
entries = []
|
|
for path in sorted(delivery_root.rglob("*")):
|
|
if path.is_file() and path.name != "manifest.json":
|
|
relative = path.relative_to(delivery_root).as_posix()
|
|
entries.append({"path": relative, "key": f"prefix/{relative}", "sizeBytes": path.stat().st_size, "sha256": "0" * 64})
|
|
(delivery_root / "manifest.json").write_text(json.dumps({"schemaVersion": 1, "kind": "directory", "fileCount": len(entries), "totalBytes": sum(e["sizeBytes"] for e in entries), "entries": entries}), encoding="utf8")
|
|
|
|
|
|
def test_migration_output_matches_the_operator_byte_for_byte(tmp_path):
|
|
delivery_root, revision_root, inputs = delivered_part(tmp_path, ["keep", "discard", "keep"])
|
|
migrated = migration_module()
|
|
payload, body = migrated.build_index(delivery_root, revision_root)
|
|
|
|
# Compare against an online package built from the same evidence. The single
|
|
# sanctioned difference is the migration provenance marker.
|
|
inputs[1]["data"]["finalAcceptance"].update({
|
|
"samplingDecision": "selected", "processingProfile": "raw_human",
|
|
"poseDelivery": {"headpose": "exclude", "handpose": "exclude"},
|
|
})
|
|
inputs[1]["data"]["capture"] = {"captureId": "CAP-1"}
|
|
inputs[1]["data"]["qualityReviews"] = [{"taskId": "task-1", "revision": 1, "decision": "approved", "reviewerUserId": "user-qc", "createdAt": "2026-01-01T00:00:00Z", "comment": "INTERNAL-ONLY"}]
|
|
artifacts = package.run("r", "p", tmp_path / "online", PART_NODE, inputs, helpers())
|
|
online = json.loads(Path(artifacts[0]["path"]).read_text(encoding="utf8"))
|
|
comparable = json.loads(json.dumps(payload))
|
|
assert comparable["lineage"].pop("migratedFrom") == "droboticflow.delivery.manifest/v6"
|
|
assert online["lineage"].pop("migratedFrom", None) is None
|
|
assert comparable == online, "migration no longer produces the same structure as an online package"
|
|
assert body == (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
|
assert payload["schemaVersion"] == "droboticflow.delivery.part-index/v1"
|
|
|
|
|
|
def test_migrated_index_records_its_provenance(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep"])
|
|
payload, _ = migration_module().build_index(delivery_root, revision_root)
|
|
assert payload["lineage"]["migratedFrom"] == "droboticflow.delivery.manifest/v6"
|
|
|
|
|
|
def test_migration_classifies_both_dispositions_and_keeps_evidence(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep", "discard"])
|
|
payload, _ = migration_module().build_index(delivery_root, revision_root)
|
|
assert [entry["sliceId"] for entry in payload["slices"]] == ["slice-000001"]
|
|
assert [entry["sliceId"] for entry in payload["discardedSlices"]] == ["slice-000002"]
|
|
assert payload["discardedSlices"][0]["classification"]["discardEvidence"]["reason"] == "PRIVATE_DISCARD_REASON"
|
|
assert payload["lineage"]["migratedFrom"] == "droboticflow.delivery.manifest/v6"
|
|
|
|
|
|
def test_migration_refuses_a_part_without_a_completion_manifest(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep"])
|
|
(delivery_root / "manifest.json").unlink()
|
|
with pytest.raises(SystemExit, match="已完成的交付目录"):
|
|
migration_module().build_index(delivery_root, revision_root)
|
|
|
|
|
|
def test_migration_refuses_an_unknown_disposition(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep", "discard"])
|
|
path = revision_root / "revision-manifest.json"
|
|
value = json.loads(path.read_text(encoding="utf8"))
|
|
value["items"][1]["deliveryDisposition"] = "guess"
|
|
path.write_text(json.dumps(value), encoding="utf8")
|
|
with pytest.raises(SystemExit, match="用途未知"):
|
|
migration_module().build_index(delivery_root, revision_root)
|
|
|
|
|
|
def test_migration_refuses_to_overwrite_a_different_existing_index(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep"])
|
|
(delivery_root / "part.json").write_text('{"different":true}', encoding="utf8")
|
|
with pytest.raises(SystemExit, match="拒绝覆盖"):
|
|
migration_module().run_migration(delivery_root, revision_root, tmp_path / "out", "part0001")
|
|
|
|
|
|
def test_migration_is_a_noop_when_the_index_is_already_delivered(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep"])
|
|
migrated = migration_module()
|
|
_, body = migrated.build_index(delivery_root, revision_root)
|
|
(delivery_root / "part.json").write_bytes(body)
|
|
report = migrated.run_migration(delivery_root, revision_root, tmp_path / "out", "part0001")
|
|
assert report["status"] == "already-migrated"
|
|
assert not (tmp_path / "out" / "part.json").exists()
|
|
|
|
|
|
def test_migration_writes_the_candidate_outside_the_delivery_directory(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep"])
|
|
report = migration_module().run_migration(delivery_root, revision_root, tmp_path / "out", "part0001")
|
|
assert report["status"] == "dry-run"
|
|
assert Path(report["written"]).is_file()
|
|
# The delivered directory is untouched: migration only adds the index.
|
|
assert not list((delivery_root / "clips").glob("**/part.json"))
|
|
assert not (delivery_root / "part.json").exists()
|
|
|
|
|
|
def test_migration_refuses_to_write_inside_the_delivery_directory(tmp_path):
|
|
delivery_root, revision_root, _ = delivered_part(tmp_path, ["keep"])
|
|
with pytest.raises(SystemExit, match="不能位于交付目录内"):
|
|
migration_module().run_migration(delivery_root, revision_root, delivery_root / "out", "part0001")
|