droboticflow/tests/backend/test_slice_discard_package.py

186 lines
12 KiB
Python

import hashlib
import json
import os
from pathlib import Path
import pytest
from backend.operators.builtin.package import operator as package
OUTPUTS = [{"id": x, "name": x} for x in ("deliveryDirectory", "deliveryManifest", "deliverySummary")]
NODE = {"outputs": OUTPUTS, "hyperparams": [{"key": "PROCESSING_PROFILE", "value": "raw_human"}]}
def helpers():
def write(run, node, directory, name, payload, output=None):
path = Path(directory) / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload), encoding="utf8")
return {"id": run + name, "name": name, "path": str(path), "data": payload, "portId": output["id"]}
def virtual(run, node, name, path, kind, size, data=None, output=None, **kwargs):
return {"id": run + name, "name": name, "path": path, "data": data, "portId": output["id"]}
return {"write_json_artifact": write, "build_virtual_artifact": virtual, "payloads_from_inputs": lambda items: [i.get("data", {}) for i in items], "source_path_from_item":lambda item,*args:Path(item.get("path") or item["source"]["path"]), "source_ref_from_artifact":lambda item:{"path":item.get("path")}}
def accepted_inputs(artifact):
sha = hashlib.sha256((Path(artifact["path"]) / "revision-manifest.json").read_bytes()).hexdigest()
revision = json.loads((Path(artifact["path"]) / "revision-manifest.json").read_text())["revision"]
artifact = {**artifact, "portId": "acceptedSliceDirectory"}
summary = {"portId": "acceptanceSummary", "data": {"decision": "approved", "processingProfile": "raw_human", "acceptedMaterializations": [{"artifactId": artifact["id"], "revision": revision, "manifestSha256": sha}], "qualityReviews": [{"decision": "approved", "comment": "PRIVATE_DISCARD_REASON"}], "finalAcceptance": {"confirmed": True, "sampled": True, "processingProfile": "raw_human", "statistics": {"notes": "PRIVATE_DISCARD_REASON"}}}}
calibration_root = Path(artifact["path"]).parent / "package-test-calibrations"
calibration_root.mkdir(exist_ok=True)
calibrations = []
for kind in ("camera", "imu"):
path = calibration_root / f"calibration_{kind}.yaml"
path.write_text(f"{kind}: synthetic-test-fixture\n")
calibrations.append({"id": kind, "name": path.name, "path": str(path), "type": "file", "portId": "calibrations"})
return [artifact, summary, *calibrations]
def fixture(tmp_path, dispositions):
root = tmp_path / "evidence"
rows = []
for i, disposition in enumerate(dispositions):
sid = f"slice-{i + 1:06d}"
folder = root / "slices" / sid
folder.mkdir(parents=True)
start_ms = (i + 1) * 1000
for camera in ("left", "right"):
(folder / f"{camera}.mp4").write_bytes(f"{sid}-{camera}".encode())
(folder / f"{camera}_pts.csv").write_text(f"frame_index,timestamp_ms,clip_time_ms\n0,{start_ms},0\n1,{start_ms+500},500\n")
(folder / "imu.csv").write_text("timestamp_ms,clip_time_ms,x,y,z,type\n" + "".join(f"{t},{t-start_ms},0,0,0,{sensor}\n" for t in (start_ms, start_ms+500) for sensor in ("accel", "gyro")))
(folder / "annotation.json").write_text(json.dumps({"actions": [] if disposition == "discard" else [{"description": "无效片段", "isInvalid": True}]}))
(folder / "synchronized-slice.json").write_text(json.dumps({"schemaVersion": "droboticflow.synchronized-slice/v1"}))
rows.append({"sliceId": sid, "directory": f"slices/{sid}", "startSecond": i, "endSecond": i + 1, "startTimestampUs": start_ms*1000, "endTimestampUs": (start_ms+1000)*1000, "deliveryDisposition": disposition, "discardEvidence": {"reason": "PRIVATE_DISCARD_REASON"} if disposition == "discard" else None})
(root / "revision-manifest.json").write_text(json.dumps({"schemaVersion": "droboticflow.slice-materialization/v3", "revision": 1, "coverage": {"mode": "full_common_range", "contiguous": True, "gapMicroseconds": 0, "overlapMicroseconds": 0, "sliceCount": len(rows)}, "items": rows}))
return accepted_inputs({"id": "accepted", "path": str(root)})
def test_discard_filters_every_file_without_hiding_review_evidence(tmp_path, monkeypatch):
inputs = fixture(tmp_path, ["keep", "discard", "keep"])
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda path: 2)
output = tmp_path / "output"
old = output / "delivery" / "clips" / "slice-000002"
old.mkdir(parents=True)
(old / "left.mp4").write_bytes(b"stale")
artifacts = package.run("r", "p", output, NODE, inputs, helpers())
delivery = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
assert {p.name for p in (delivery / "clips").iterdir()} == {"slice-000001", "slice-000003"}
manifest = json.loads((delivery / "packages-manifest.json").read_text())
assert manifest["coverage"]["durationSeconds"] == 2
assert manifest["sliceCount"] == 2 and manifest["schemaVersion"].endswith("/v6")
assert (Path(inputs[0]["path"]) / "slices/slice-000002/left.mp4").is_file()
assert all("PRIVATE_DISCARD_REASON" not in p.read_text() for p in delivery.rglob("*.json"))
assert json.loads((delivery / "clips/slice-000001/annotation.json").read_text())["actions"][0]["description"] == "无效片段"
again = package.run("r", "p", output, NODE, inputs, helpers())
new_directory = Path(next(a["path"] for a in again if a["portId"] == "deliveryDirectory"))
assert new_directory != delivery and not (new_directory / "clips/slice-000002").exists()
def test_all_discard_emits_only_internal_outcome_not_empty_delivery(tmp_path):
inputs = fixture(tmp_path, ["discard", "discard", "discard"])
artifacts = package.run("r", "p", tmp_path / "output", NODE, inputs, helpers())
assert len(artifacts) == 1 and artifacts[0]["portId"] == "deliverySummary"
assert artifacts[0]["data"]["outcome"] == "no_deliverables"
assert not list((tmp_path / "output").glob("delivery*"))
assert len(list((Path(inputs[0]["path"]) / "slices").iterdir())) == 3
@pytest.mark.parametrize("partial", [False, True])
def test_blank_annotations_keep_all_delivery_media(tmp_path, monkeypatch, partial):
inputs = fixture(tmp_path, ["keep", "keep", "keep"])
source = Path(inputs[0]["path"])
for i in range(1, 4):
actions = [{"description": "拿起物品", "clipStartMs": 250, "clipEndMs": 500}] if partial and i == 1 else []
(source / f"slices/slice-{i:06d}/annotation.json").write_text(json.dumps({"actions": actions, "descriptions": actions}))
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda path: 2)
artifacts = package.run("r", "p", tmp_path / "output", NODE, inputs, helpers())
delivery = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
manifest = json.loads((delivery / "packages-manifest.json").read_text())
assert manifest["sliceCount"] == 3
assert manifest["coverage"]["durationSeconds"] == 3
for i in range(1, 4):
folder = delivery / f"clips/slice-{i:06d}"
assert all((folder / name).is_file() for name in ("left.mp4", "right.mp4", "left_pts.csv", "right_pts.csv", "imu.csv"))
exported = json.loads((folder / "annotation.json").read_text())["actions"]
assert len(exported) == (1 if partial and i == 1 else 0)
assert all(not event.get("isInvalid") for event in exported)
@pytest.mark.parametrize("failure", ["stale", "unapproved", "missing-reason", "unknown-policy"])
def test_discard_package_requires_exact_approved_decision(tmp_path, failure):
inputs = fixture(tmp_path, ["keep", "discard", "keep"])
if failure == "stale":
inputs[1]["data"]["acceptedMaterializations"][0]["revision"] = 9
elif failure == "unapproved":
inputs[1]["data"]["finalAcceptance"]["confirmed"] = False
else:
path = Path(inputs[0]["path"]) / "revision-manifest.json"
value = json.loads(path.read_text())
if failure == "missing-reason":
value["items"][1]["discardEvidence"] = {}
else:
value["items"][1]["deliveryDisposition"] = "guess"
path.write_text(json.dumps(value))
inputs = accepted_inputs(inputs[0])
with pytest.raises(RuntimeError):
package.run("r", "p", tmp_path / "output", NODE, inputs, helpers())
def test_real_60_second_materialization_to_delivery(tmp_path):
context_path = os.environ.get("FANYU_DISCARD_MATERIALIZATION_CONTEXT")
if not context_path:
pytest.skip("opt-in Go materialization fixture")
contexts = json.loads(Path(context_path).read_text())
for mode, expected_count in [("keep", 3), ("mixed", 2), ("discard", 0), ("blank", 3), ("partial", 3)]:
inputs = accepted_inputs(contexts[mode]["artifact"])
artifacts = package.run("r", "p", tmp_path / mode, NODE, inputs, helpers())
directories = [a for a in artifacts if a["portId"] == "deliveryDirectory"]
if expected_count == 0:
assert not directories and artifacts[0]["data"]["outcome"] == "no_deliverables"
continue
root = Path(directories[0]["path"])
manifest = json.loads((root / "packages-manifest.json").read_text())
assert len(manifest["items"]) == expected_count
for entry in manifest["items"]:
for f in entry["files"]:
assert hashlib.sha256((root / entry["directory"] / f["name"]).read_bytes()).hexdigest() == f["sha256"]
assert package._ffprobe_frame_count(root / entry["directory"] / "left.mp4") == 80
assert package._ffprobe_frame_count(root / entry["directory"] / "right.mp4") == 80
if mode == "mixed":
assert manifest["coverage"]["durationSeconds"] == 40
assert not (root / "clips/slice-000002").exists()
if mode in ("blank", "partial"):
actions = [json.loads((root / entry["directory"] / "annotation.json").read_text())["actions"] for entry in manifest["items"]]
assert [len(a) for a in actions] == ([0, 0, 0] if mode == "blank" else [1, 0, 0])
assert sum(entry["endSecond"] - entry["startSecond"] for entry in manifest["items"]) == 60
if mode == "partial":
assert actions[0][0]["clipStartMs"] == 5000
assert actions[0][0]["clipEndMs"] == 10000
@pytest.mark.parametrize("actions", [
[{"id": "whole-blank", "status": "pending", "description": "", "isInvalid": False, "clipStartMs": 0, "clipEndMs": 1000}],
[{"id": "blank-left", "status": "pending", "description": "", "isInvalid": False, "clipStartMs": 0, "clipEndMs": 500},
{"id": "blank-right", "status": "pending", "description": "", "isInvalid": False, "clipStartMs": 500, "clipEndMs": 1000}],
[{"id": "blank", "status": "pending", "description": "", "isInvalid": False, "clipStartMs": 0, "clipEndMs": 250},
{"id": "described", "status": "described", "description": "拿取商品", "isInvalid": False, "clipStartMs": 250, "clipEndMs": 500},
{"id": "invalid", "status": "invalid", "description": "无效片段", "isInvalid": True, "clipStartMs": 500, "clipEndMs": 1000}],
], ids=["whole-blank", "partitioned-blank", "mixed-judgments"])
def test_delivery_retains_explicit_blank_action_records(tmp_path, monkeypatch, actions):
inputs = fixture(tmp_path, ["keep"])
source = Path(inputs[0]["path"]) / "slices/slice-000001/annotation.json"
annotation = {"schemaVersion": "droboticflow.timestamped-description/v3", "actions": actions, "descriptions": actions}
source.write_text(json.dumps(annotation), encoding="utf8")
monkeypatch.setattr(package, "_ffprobe_frame_count", lambda path: 2)
artifacts = package.run("r", "p", tmp_path / "output", NODE, inputs, helpers())
delivery = Path(next(a["path"] for a in artifacts if a["portId"] == "deliveryDirectory"))
exported = json.loads((delivery / "clips/slice-000001/annotation.json").read_text())
assert exported["actions"] == actions
assert exported["descriptions"] == actions
manifest = json.loads((delivery / "packages-manifest.json").read_text())
assert manifest["sliceCount"] == 1
assert manifest["coverage"]["durationSeconds"] == 1