208 lines
12 KiB
Python
208 lines
12 KiB
Python
"""Cross-language, real-media source-index revision -> both export contracts."""
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from backend.operators.builtin.package import operator as package
|
|
from tests.backend.test_slice_discard_package import accepted_inputs, fixture as physical_slice_fixture, NODE
|
|
from tests.backend.test_part_index_package import helpers
|
|
|
|
|
|
@pytest.fixture
|
|
def source_inputs(tmp_path):
|
|
fixture_dir = os.environ.get("FANYU_SOURCE_INDEX_FIXTURE_DIR")
|
|
if not fixture_dir:
|
|
pytest.skip("generate local Go source-index fixture first")
|
|
context = json.loads(Path((Path(fixture_dir) / "latest-context.txt").read_text()).read_text())
|
|
old_workspace = Path(context["workspace"])
|
|
workspace = tmp_path / "workspace"
|
|
shutil.copytree(old_workspace, workspace)
|
|
artifact = context["artifact"]
|
|
artifact["path"] = str(workspace / Path(artifact["path"]).relative_to(old_workspace))
|
|
root = Path(artifact["path"])
|
|
manifest = json.loads((root / "revision-manifest.json").read_text())
|
|
for source in manifest["sources"].values():
|
|
source["path"] = str(workspace / Path(source["path"]).relative_to(old_workspace))
|
|
(root / "revision-manifest.json").write_text(json.dumps(manifest))
|
|
return accepted_inputs(artifact)
|
|
|
|
|
|
def test_source_index_real_media_legacy_export_keeps_exact_frames_and_discards(tmp_path, source_inputs):
|
|
artifacts = package.run("test", "package", tmp_path / "output", NODE, source_inputs, helpers())
|
|
root = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
|
|
assert {p.name for p in (root / "clips").iterdir()} == {"slice-000001", "slice-000003"}
|
|
manifest = json.loads((root / "packages-manifest.json").read_text())
|
|
assert manifest["sliceCount"] == 2
|
|
for item in manifest["items"]:
|
|
folder = root / item["directory"]
|
|
for side in ("left", "right"):
|
|
assert package._ffprobe_frame_count(folder / (side + ".mp4")) == 8
|
|
with (folder / (side + "_pts.csv")).open() as f:
|
|
rows = list(csv.DictReader(f))
|
|
assert len(rows) == 8
|
|
assert rows[0]["frame_index"] == "0"
|
|
assert rows[0]["clip_time_ms"] == "0.000"
|
|
with (folder / "imu.csv").open() as f:
|
|
assert len(list(csv.DictReader(f))) == 16
|
|
sync = json.loads((folder / "synchronized-slice.json").read_text())
|
|
assert "sourceMediaOffsets" not in sync and "sourceIndexSchema" not in sync
|
|
assert all("synthetic-discard" not in p.read_text() for p in root.rglob("*.json"))
|
|
|
|
|
|
@pytest.mark.parametrize("v3", [False, True])
|
|
def test_source_index_json_delivery_never_decodes_or_hashes_full_videos(tmp_path, source_inputs, monkeypatch, v3):
|
|
node = {"outputs": NODE["outputs"] + [{"id": "deliveryPartIndex", "name": "deliveryPartIndex"}],
|
|
"hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}, {"key": "SOURCE_INDEX_DELIVERY", "value": "true"}]}
|
|
if v3:
|
|
node["hyperparams"].append({"key": "DELIVERY_JSON_V3", "value": "true"})
|
|
source_inputs[1]["data"]["capture"] = {"captureId": "capture-A/part0001", "prefix": "incoming/capture-A/part0001"}
|
|
def forbidden(*args, **kwargs):
|
|
raise AssertionError("JSON-only export must not transcode")
|
|
monkeypatch.setattr(package.subprocess, "run", forbidden)
|
|
original_sha = package._sha256_file
|
|
def metadata_only(path):
|
|
assert Path(path).suffix != ".mp4", "JSON export rehashed source media"
|
|
return original_sha(path)
|
|
monkeypatch.setattr(package, "_sha256_file", metadata_only)
|
|
artifacts = package.run("test", "package", tmp_path / "output", node, source_inputs, helpers())
|
|
assert len(artifacts) == 1
|
|
index_path = Path(artifacts[0]["path"]) / "part.json" if v3 else Path(artifacts[0]["path"])
|
|
payload = json.loads(index_path.read_text())
|
|
assert payload["schemaVersion"] == ("droboticflow.delivery.part-index/v3" if v3 else "droboticflow.delivery.part-index/v2")
|
|
assert len(payload["sources"]) == 5
|
|
assert [item["sliceId"] for item in payload["slices"]] == ["slice-000001", "slice-000003"]
|
|
assert payload["discardedSlices"][0]["sourceFrames"] == {"left": [8, 24], "right": [8, 24]}
|
|
assert not list((tmp_path / "output").rglob("*.mp4"))
|
|
assert not list((tmp_path / "output").rglob("*.csv"))
|
|
assert "/workspace/" not in json.dumps(payload)
|
|
|
|
|
|
def test_source_index_export_rejects_foreign_source_or_corrupted_metadata(tmp_path, source_inputs):
|
|
root = Path(source_inputs[0]["path"])
|
|
manifest_path = root / "revision-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text())
|
|
manifest["sources"]["leftVideo"]["path"] = "/tmp/other-customer.mp4"
|
|
manifest_path.write_text(json.dumps(manifest))
|
|
inputs = accepted_inputs(source_inputs[0])
|
|
with pytest.raises(RuntimeError, match="pinned source"):
|
|
package.run("test", "package", tmp_path / "output", NODE, inputs, helpers())
|
|
|
|
|
|
def test_source_index_json_delivery_requires_new_contract_opt_in(tmp_path, source_inputs):
|
|
node = {"outputs": NODE["outputs"], "hyperparams": NODE["hyperparams"] + [{"key": "PART_INDEX_DELIVERY", "value": "true"}]}
|
|
with pytest.raises(RuntimeError, match="explicit SOURCE_INDEX_DELIVERY"):
|
|
package.run("test", "package", tmp_path / "output", node, source_inputs, helpers())
|
|
|
|
|
|
def test_source_index_export_keeps_kept_slice_with_zero_imu_samples(tmp_path, monkeypatch):
|
|
"""Review 2026-10-18: a kept slice inside an IMU gap has sampleCount 0 in
|
|
its certificate; the export must accept it and keep a header-only imu.csv."""
|
|
source_root = tmp_path / "revision"
|
|
imu = tmp_path / "imu.csv"
|
|
imu.write_text("timestamp_us,x,y,z,type\n1000,0,0,1,accel\n1000,0,1,0,gyro\n1500,0,0,1,accel\n1500,0,1,0,gyro\n")
|
|
kept = [
|
|
{"directory": "slices/slice-000001", "startTimestampUs": 1000, "endTimestampUs": 2000, "sourceFrames": {"left": [0, 2], "right": [0, 2]}},
|
|
{"directory": "slices/slice-000002", "startTimestampUs": 2000, "endTimestampUs": 3000, "sourceFrames": {"left": [2, 4], "right": [2, 4]}},
|
|
]
|
|
for item, samples in zip(kept, (4, 0)):
|
|
folder = source_root / item["directory"]
|
|
folder.mkdir(parents=True)
|
|
(folder / "annotation.json").write_text("{}")
|
|
(folder / "synchronized-slice.json").write_text(json.dumps({"imu": {"sampleCount": samples}}))
|
|
pts = [1000, 1500, 2000, 2500]
|
|
monkeypatch.setattr(package, "_source_index_files", lambda *_: {"leftVideo": tmp_path / "l.mp4", "rightVideo": tmp_path / "r.mp4", "leftPts": tmp_path / "lp.csv", "rightPts": tmp_path / "rp.csv", "imu": imu})
|
|
monkeypatch.setattr(package, "_read_pts", lambda _: pts)
|
|
monkeypatch.setattr(package.subprocess, "run", lambda *a, **k: type("Done", (), {"returncode": 0})())
|
|
manifest = {"sourceDurations": {"left": 2.0, "right": 2.0}}
|
|
|
|
target = package._source_index_physical_export(source_root, manifest, kept, tmp_path)
|
|
|
|
with (target / "slices/slice-000001/imu.csv").open() as f:
|
|
assert len(list(csv.DictReader(f))) == 4
|
|
with (target / "slices/slice-000002/imu.csv").open() as f:
|
|
reader = csv.DictReader(f)
|
|
assert reader.fieldnames == ["timestamp_ms", "clip_time_ms", "x", "y", "z", "type"]
|
|
assert list(reader) == []
|
|
|
|
|
|
def test_source_index_export_still_rejects_imu_count_mismatch(tmp_path, monkeypatch):
|
|
source_root = tmp_path / "revision"
|
|
imu = tmp_path / "imu.csv"
|
|
imu.write_text("timestamp_us,x,y,z,type\n1000,0,0,1,accel\n1000,0,1,0,gyro\n")
|
|
kept = [{"directory": "slices/slice-000001", "startTimestampUs": 1000, "endTimestampUs": 2000, "sourceFrames": {"left": [0, 2], "right": [0, 2]}}]
|
|
folder = source_root / kept[0]["directory"]
|
|
folder.mkdir(parents=True)
|
|
(folder / "annotation.json").write_text("{}")
|
|
(folder / "synchronized-slice.json").write_text(json.dumps({"imu": {"sampleCount": 0}}))
|
|
monkeypatch.setattr(package, "_source_index_files", lambda *_: {"leftVideo": tmp_path / "l.mp4", "rightVideo": tmp_path / "r.mp4", "leftPts": tmp_path / "lp.csv", "rightPts": tmp_path / "rp.csv", "imu": imu})
|
|
monkeypatch.setattr(package, "_read_pts", lambda _: [1000, 1500])
|
|
monkeypatch.setattr(package.subprocess, "run", lambda *a, **k: type("Done", (), {"returncode": 0})())
|
|
with pytest.raises(RuntimeError, match="IMU reconstruction mismatch"):
|
|
package._source_index_physical_export(source_root, {"sourceDurations": {"left": 1.0, "right": 1.0}}, kept, tmp_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def zero_imu_source_inputs(source_inputs):
|
|
"""A real-media source-index slice inside a legitimate sensor gap."""
|
|
root = Path(source_inputs[0]["path"])
|
|
manifest_path = root / "revision-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text())
|
|
kept = next(item for item in manifest["items"] if item["deliveryDisposition"] == "keep")
|
|
source = manifest["sources"]["imu"]
|
|
imu = Path(source["path"])
|
|
with imu.open(newline="") as handle:
|
|
reader = csv.DictReader(handle)
|
|
fields, rows = reader.fieldnames, list(reader)
|
|
with imu.open("w", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n")
|
|
writer.writeheader()
|
|
writer.writerows(row for row in rows if not kept["startTimestampUs"] <= int(row["timestamp_us"]) < kept["endTimestampUs"])
|
|
source["bytes"] = imu.stat().st_size
|
|
source["sha256"] = hashlib.sha256(imu.read_bytes()).hexdigest()
|
|
sync_path = root / kept["directory"] / "synchronized-slice.json"
|
|
sync = json.loads(sync_path.read_text())
|
|
sync["imu"]["sampleCount"] = 0
|
|
sync_path.write_text(json.dumps(sync))
|
|
kept["files"]["synchronization"]["sha256"] = hashlib.sha256(sync_path.read_bytes()).hexdigest()
|
|
manifest_path.write_text(json.dumps(manifest))
|
|
return accepted_inputs(source_inputs[0])
|
|
|
|
|
|
def test_source_index_full_delivery_accepts_declared_zero_imu(tmp_path, zero_imu_source_inputs):
|
|
artifacts = package.run("test", "package", tmp_path / "output", NODE, zero_imu_source_inputs, helpers())
|
|
root = Path(next(item["path"] for item in artifacts if item["portId"] == "deliveryDirectory"))
|
|
with (root / "clips/slice-000001/imu.csv").open() as handle:
|
|
reader = csv.DictReader(handle)
|
|
assert reader.fieldnames == ["timestamp_ms", "clip_time_ms", "x", "y", "z", "type"]
|
|
assert list(reader) == []
|
|
with (root / "clips/slice-000003/imu.csv").open() as handle:
|
|
assert len(list(csv.DictReader(handle))) == 16
|
|
|
|
|
|
def test_source_index_full_delivery_rejects_false_zero_imu_count(tmp_path, zero_imu_source_inputs):
|
|
root = Path(zero_imu_source_inputs[0]["path"])
|
|
manifest_path = root / "revision-manifest.json"
|
|
manifest = json.loads(manifest_path.read_text())
|
|
kept = next(item for item in manifest["items"] if item["deliveryDisposition"] == "keep")
|
|
sync_path = root / kept["directory"] / "synchronized-slice.json"
|
|
sync = json.loads(sync_path.read_text())
|
|
sync["imu"]["sampleCount"] = 1
|
|
sync_path.write_text(json.dumps(sync))
|
|
kept["files"]["synchronization"]["sha256"] = hashlib.sha256(sync_path.read_bytes()).hexdigest()
|
|
manifest_path.write_text(json.dumps(manifest))
|
|
with pytest.raises(RuntimeError, match="IMU reconstruction mismatch"):
|
|
package.run("test", "package", tmp_path / "output", NODE, accepted_inputs(zero_imu_source_inputs[0]), helpers())
|
|
|
|
|
|
def test_legacy_physical_delivery_still_rejects_empty_imu(tmp_path, monkeypatch):
|
|
inputs = physical_slice_fixture(tmp_path, ["keep"])
|
|
root = Path(inputs[0]["path"])
|
|
(root / "slices/slice-000001/imu.csv").write_text("timestamp_ms,clip_time_ms,x,y,z,type\n")
|
|
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda _: 2)
|
|
with pytest.raises(RuntimeError, match="paired accel/gyro"):
|
|
package.run("test", "package", tmp_path / "output", NODE, inputs, helpers())
|