droboticflow/tests/backend/test_source_index_package.py

208 lines
12 KiB
Python

"""Cross-language, real-media source-index revision -> both export contracts."""
import csv
import hashlib
import json
import os
import shutil
from pathlib import Path
import pytest
from backend.operators.builtin.package import operator as package
from tests.backend.test_slice_discard_package import accepted_inputs, fixture as physical_slice_fixture, NODE
from tests.backend.test_part_index_package import helpers
@pytest.fixture
def source_inputs(tmp_path):
fixture_dir = os.environ.get("FANYU_SOURCE_INDEX_FIXTURE_DIR")
if not fixture_dir:
pytest.skip("generate local Go source-index fixture first")
context = json.loads(Path((Path(fixture_dir) / "latest-context.txt").read_text()).read_text())
old_workspace = Path(context["workspace"])
workspace = tmp_path / "workspace"
shutil.copytree(old_workspace, workspace)
artifact = context["artifact"]
artifact["path"] = str(workspace / Path(artifact["path"]).relative_to(old_workspace))
root = Path(artifact["path"])
manifest = json.loads((root / "revision-manifest.json").read_text())
for source in manifest["sources"].values():
source["path"] = str(workspace / Path(source["path"]).relative_to(old_workspace))
(root / "revision-manifest.json").write_text(json.dumps(manifest))
return accepted_inputs(artifact)
def test_source_index_real_media_legacy_export_keeps_exact_frames_and_discards(tmp_path, source_inputs):
artifacts = package.run("test", "package", tmp_path / "output", NODE, source_inputs, helpers())
root = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
assert {p.name for p in (root / "clips").iterdir()} == {"slice-000001", "slice-000003"}
manifest = json.loads((root / "packages-manifest.json").read_text())
assert manifest["sliceCount"] == 2
for item in manifest["items"]:
folder = root / item["directory"]
for side in ("left", "right"):
assert package._ffprobe_frame_count(folder / (side + ".mp4")) == 8
with (folder / (side + "_pts.csv")).open() as f:
rows = list(csv.DictReader(f))
assert len(rows) == 8
assert rows[0]["frame_index"] == "0"
assert rows[0]["clip_time_ms"] == "0.000"
with (folder / "imu.csv").open() as f:
assert len(list(csv.DictReader(f))) == 16
sync = json.loads((folder / "synchronized-slice.json").read_text())
assert "sourceMediaOffsets" not in sync and "sourceIndexSchema" not in sync
assert all("synthetic-discard" not in p.read_text() for p in root.rglob("*.json"))
@pytest.mark.parametrize("v3", [False, True])
def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch, v3):
node = {"outputs": NODE["outputs"] + [{"id": "deliveryPartIndex", "name": "deliveryPartIndex"}],
"hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}, {"key": "SOURCE_INDEX_DELIVERY", "value": "true"}]}
if v3:
node["hyperparams"].append({"key": "DELIVERY_JSON_V3", "value": "true"})
source_inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001"}
def forbidden(*args, **kwargs):
raise AssertionError("JSON-only export must not transcode")
monkeypatch.setattr(package.subprocess, "run", forbidden)
original_sha = package._sha256_file
def metadata_only(path):
assert Path(path).suffix != ".mp4", "JSON export rehashed source media"
return original_sha(path)
monkeypatch.setattr(package, "_sha256_file", metadata_only)
artifacts = package.run("test", "package", tmp_path / "output", node, source_inputs, helpers())
assert len(artifacts) == 1
index_path = Path(artifacts[0]["path"]) / "part.json" if v3 else Path(artifacts[0]["path"])
payload = json.loads(index_path.read_text())
assert payload["schemaVersion"] == ("droboticflow.delivery.part-index/v3" if v3 else "droboticflow.delivery.part-index/v2")
assert len(payload["sources"]) == 5
assert [item["sliceId"] for item in payload["slices"]] == ["slice-000001", "slice-000003"]
assert payload["discardedSlices"][0]["sourceFrames"] == {"left": [8, 24], "right": [8, 24]}
assert not list((tmp_path / "output").rglob("*.mp4"))
assert not list((tmp_path / "output").rglob("*.csv"))
assert "/workspace/" not in json.dumps(payload)
def test_source_index_export_rejects_foreign_source_or_corrupted_metadata(tmp_path, source_inputs):
root = Path(source_inputs[0]["path"])
manifest_path = root / "revision-manifest.json"
manifest = json.loads(manifest_path.read_text())
manifest["sources"]["leftVideo"]["path"] = "/tmp/other-customer.mp4"
manifest_path.write_text(json.dumps(manifest))
inputs = accepted_inputs(source_inputs[0])
with pytest.raises(RuntimeError, match="pinned source"):
package.run("test", "package", tmp_path / "output", NODE, inputs, helpers())
def test_source_index_json_delivery_requires_new_contract_opt_in(tmp_path, source_inputs):
node = {"outputs": NODE["outputs"], "hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}]}
with pytest.raises(RuntimeError, match="explicit SOURCE_INDEX_DELIVERY"):
package.run("test", "package", tmp_path / "output", node, source_inputs, helpers())
def test_source_index_export_keeps_kept_slice_with_zero_imu_samples(tmp_path, monkeypatch):
"""Review 2026-10-18: a kept slice inside an IMU gap has sampleCount 0 in
its certificate; the export must accept it and keep a header-only imu.csv."""
source_root = tmp_path / "revision"
imu = tmp_path / "imu.csv"
imu.write_text("timestamp_us,x,y,z,type\n1000,0,0,1,accel\n1000,0,1,0,gyro\n1500,0,0,1,accel\n1500,0,1,0,gyro\n")
kept = [
{"directory": "slices/slice-000001", "startTimestampUs": 1000, "endTimestampUs": 2000, "sourceFrames": {"left": [0, 2], "right": [0, 2]}},
{"directory": "slices/slice-000002", "startTimestampUs": 2000, "endTimestampUs": 3000, "sourceFrames": {"left": [2, 4], "right": [2, 4]}},
]
for item, samples in zip(kept, (4, 0)):
folder = source_root / item["directory"]
folder.mkdir(parents=True)
(folder / "annotation.json").write_text("{}")
(folder / "synchronized-slice.json").write_text(json.dumps({"imu": {"sampleCount": samples}}))
pts = [1000, 1500, 2000, 2500]
monkeypatch.setattr(package, "_source_index_files", lambda *_: {"leftVideo": tmp_path / "l.mp4", "rightVideo": tmp_path / "r.mp4", "leftPts": tmp_path / "lp.csv", "rightPts": tmp_path / "rp.csv", "imu": imu})
monkeypatch.setattr(package, "_read_pts", lambda _: pts)
monkeypatch.setattr(package.subprocess, "run", lambda *a, **k: type("Done", (), {"returncode": 0})())
manifest = {"sourceDurations": {"left": 2.0, "right": 2.0}}
target = package._source_index_physical_export(source_root, manifest, kept, tmp_path)
with (target / "slices/slice-000001/imu.csv").open() as f:
assert len(list(csv.DictReader(f))) == 4
with (target / "slices/slice-000002/imu.csv").open() as f:
reader = csv.DictReader(f)
assert reader.fieldnames == ["timestamp_ms", "clip_time_ms", "x", "y", "z", "type"]
assert list(reader) == []
def test_source_index_export_still_rejects_imu_count_mismatch(tmp_path, monkeypatch):
source_root = tmp_path / "revision"
imu = tmp_path / "imu.csv"
imu.write_text("timestamp_us,x,y,z,type\n1000,0,0,1,accel\n1000,0,1,0,gyro\n")
kept = [{"directory": "slices/slice-000001", "startTimestampUs": 1000, "endTimestampUs": 2000, "sourceFrames": {"left": [0, 2], "right": [0, 2]}}]
folder = source_root / kept[0]["directory"]
folder.mkdir(parents=True)
(folder / "annotation.json").write_text("{}")
(folder / "synchronized-slice.json").write_text(json.dumps({"imu": {"sampleCount": 0}}))
monkeypatch.setattr(package, "_source_index_files", lambda *_: {"leftVideo": tmp_path / "l.mp4", "rightVideo": tmp_path / "r.mp4", "leftPts": tmp_path / "lp.csv", "rightPts": tmp_path / "rp.csv", "imu": imu})
monkeypatch.setattr(package, "_read_pts", lambda _: [1000, 1500])
monkeypatch.setattr(package.subprocess, "run", lambda *a, **k: type("Done", (), {"returncode": 0})())
with pytest.raises(RuntimeError, match="IMU reconstruction mismatch"):
package._source_index_physical_export(source_root, {"sourceDurations": {"left": 1.0, "right": 1.0}}, kept, tmp_path)
@pytest.fixture
def zero_imu_source_inputs(source_inputs):
"""A real-media source-index slice inside a legitimate sensor gap."""
root = Path(source_inputs[0]["path"])
manifest_path = root / "revision-manifest.json"
manifest = json.loads(manifest_path.read_text())
kept = next(item for item in manifest["items"] if item["deliveryDisposition"] == "keep")
source = manifest["sources"]["imu"]
imu = Path(source["path"])
with imu.open(newline="") as handle:
reader = csv.DictReader(handle)
fields, rows = reader.fieldnames, list(reader)
with imu.open("w", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n")
writer.writeheader()
writer.writerows(row for row in rows if not kept["startTimestampUs"] <= int(row["timestamp_us"]) < kept["endTimestampUs"])
source["bytes"] = imu.stat().st_size
source["sha256"] = hashlib.sha256(imu.read_bytes()).hexdigest()
sync_path = root / kept["directory"] / "synchronized-slice.json"
sync = json.loads(sync_path.read_text())
sync["imu"]["sampleCount"] = 0
sync_path.write_text(json.dumps(sync))
kept["files"]["synchronization"]["sha256"] = hashlib.sha256(sync_path.read_bytes()).hexdigest()
manifest_path.write_text(json.dumps(manifest))
return accepted_inputs(source_inputs[0])
def test_source_index_full_delivery_accepts_declared_zero_imu(tmp_path, zero_imu_source_inputs):
artifacts = package.run("test", "package", tmp_path / "output", NODE, zero_imu_source_inputs, helpers())
root = Path(next(item["path"] for item in artifacts if item["portId"] == "deliveryDirectory"))
with (root / "clips/slice-000001/imu.csv").open() as handle:
reader = csv.DictReader(handle)
assert reader.fieldnames == ["timestamp_ms", "clip_time_ms", "x", "y", "z", "type"]
assert list(reader) == []
with (root / "clips/slice-000003/imu.csv").open() as handle:
assert len(list(csv.DictReader(handle))) == 16
def test_source_index_full_delivery_rejects_false_zero_imu_count(tmp_path, zero_imu_source_inputs):
root = Path(zero_imu_source_inputs[0]["path"])
manifest_path = root / "revision-manifest.json"
manifest = json.loads(manifest_path.read_text())
kept = next(item for item in manifest["items"] if item["deliveryDisposition"] == "keep")
sync_path = root / kept["directory"] / "synchronized-slice.json"
sync = json.loads(sync_path.read_text())
sync["imu"]["sampleCount"] = 1
sync_path.write_text(json.dumps(sync))
kept["files"]["synchronization"]["sha256"] = hashlib.sha256(sync_path.read_bytes()).hexdigest()
manifest_path.write_text(json.dumps(manifest))
with pytest.raises(RuntimeError, match="IMU reconstruction mismatch"):
package.run("test", "package", tmp_path / "output", NODE, accepted_inputs(zero_imu_source_inputs[0]), helpers())
def test_legacy_physical_delivery_still_rejects_empty_imu(tmp_path, monkeypatch):
inputs = physical_slice_fixture(tmp_path, ["keep"])
root = Path(inputs[0]["path"])
(root / "slices/slice-000001/imu.csv").write_text("timestamp_ms,clip_time_ms,x,y,z,type\n")
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda _: 2)
with pytest.raises(RuntimeError, match="paired accel/gyro"):
package.run("test", "package", tmp_path / "output", NODE, inputs, helpers())