droboticflow/tests/backend/test_qwen_behavior_rough_cut.py
2026-08-27 12:02:59 +08:00

400 lines
19 KiB
Python

"""Unit contracts for the PTS-grounded Qwen behaviour rough-cut builtin.
These tests deliberately exercise only parsing, matching and virtual-artifact
invariants. They do not stand in for the real CUDA Qwen3-VL acceptance run.
"""
from __future__ import annotations
import json
import pathlib
import tempfile
import unittest
from backend.operators.builtin.qwen_behavior_rough_cut import operator
ROOT = pathlib.Path(__file__).resolve().parents[2]
class TestQwenBehaviorRoughCut(unittest.TestCase):
def test_reference_script_sha_and_no_legacy_temporal_parameters(self) -> None:
source = (ROOT / "backend/operators/builtin/qwen_behavior_rough_cut/operator.py").read_text(encoding="utf-8")
self.assertIn("2cf5fdb34fc11911dbcdbd4c51965d9cfe17239f862ebd9cad2e18078f90afc2", source)
self.assertNotIn('"WINDOW_SECONDS"', source)
self.assertNotIn('"STRIDE_SECONDS"', source)
self.assertNotIn('"OVERLAP_SECONDS"', source)
self.assertIn("technicalOnly", source)
self.assertIn("sourceIsFullVideo", source)
self.assertIn('"video": planned_frame_paths', source)
self.assertIn('process_vision_info(conversation, return_video_kwargs=True)', source)
self.assertIn("start_frame_id", source)
self.assertIn("_normalize_frame_behavior", source)
self.assertNotIn('{"type": "video", "video": str(video_path)', source)
def test_pts_csv_requires_strictly_increasing_timestamp_us(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = pathlib.Path(directory)
valid = root / "capture_camera_left_part0001_pts.csv"
valid.write_text("timestamp_us\n1000000\n1040000\n1080000\n", encoding="utf-8")
self.assertEqual(operator._read_pts_csv(valid), [1000000, 1040000, 1080000])
duplicate = root / "duplicate_pts.csv"
duplicate.write_text("timestamp_us\n1000000\n1000000\n", encoding="utf-8")
with self.assertRaisesRegex(RuntimeError, "strictly increasing"):
operator._read_pts_csv(duplicate)
def test_model_times_must_be_visible_pts_and_map_from_pts_origin(self) -> None:
mapped = operator._normalize_behavior(
{
"behavior": "员工拿起商品",
"object_change": "商品从货架移到手中",
"start_timestamp": "PTS_US=1040000",
"end_timestamp": "PTS_US=1120000",
},
{1040000, 1120000},
{1000000: 0, 1040000: 1, 1080000: 2, 1120000: 3},
1000000,
25.0,
50000,
)
self.assertEqual((mapped["start_second"], mapped["end_second"]), (0.04, 0.12))
self.assertEqual((mapped["start_frame_index"], mapped["end_frame_index"]), (1, 3))
self.assertEqual((mapped["start_pts_offset_second"], mapped["end_pts_offset_second"]), (0.04, 0.12))
with self.assertRaisesRegex(RuntimeError, "bounded PTS OCR tolerance"):
operator._normalize_behavior(
{
"behavior": "员工拿起商品",
"object_change": "",
"start_timestamp": "PTS_US=900000",
"end_timestamp": "PTS_US=1120000",
},
{1040000, 1120000},
{1000000: 0, 1040000: 1, 1080000: 2, 1120000: 3},
1000000,
25.0,
1000,
)
with self.assertRaisesRegex(RuntimeError, "exact PTS_US overlay"):
operator._parse_overlay_timestamp("1120000")
rounded = operator._normalize_behavior(
{"behavior": "员工拿起商品", "object_change": "商品移动", "start_timestamp": "PTS_US=1040500", "end_timestamp": "PTS_US=1119500"},
{1040000, 1120000},
{1040000: 1, 1120000: 3},
1000000,
25.0,
50000,
)
self.assertEqual((rounded["start_us"], rounded["end_us"]), (1040000, 1120000))
self.assertEqual((rounded["start_resolution_error_us"], rounded["end_resolution_error_us"]), (500, 500))
one_frame = operator._normalize_behavior(
{"behavior": "短暂触碰", "object_change": "物体轻微移动", "start_timestamp": "PTS_US=1040000", "end_timestamp": "PTS_US=1040500"},
{1040000, 1073333, 1106666},
{1040000: 1, 1073333: 2, 1106666: 3},
1000000,
30.0,
50000,
)
self.assertEqual((one_frame["start_us"], one_frame["end_us"]), (1040000, 1073333))
self.assertEqual((one_frame["start_frame_index"], one_frame["end_frame_index"]), (1, 2))
def test_adjacent_same_behavior_merges_without_duration_heuristic(self) -> None:
events = [
{
"behavior": "拿起商品",
"object_change": "商品离开货架",
"start_timestamp": "PTS_US=1000000",
"end_timestamp": "PTS_US=1100000",
"start_us": 1000000,
"end_us": 1100000,
"model_start_us": 1000000,
"model_end_us": 1100000,
"start_resolution_error_us": 0,
"end_resolution_error_us": 0,
"start_frame_index": 0,
"end_frame_index": 1,
"start_second": 0.0,
"end_second": 0.1,
"start_pts_offset_second": 0.0,
"end_pts_offset_second": 0.1,
"technical_block_indexes": [1],
"raw_event_indexes": [1],
"raw_response_sha256": "a",
},
{
"behavior": "拿起商品",
"object_change": "商品在手中",
"start_timestamp": "PTS_US=1200000",
"end_timestamp": "PTS_US=1300000",
"start_us": 1200000,
"end_us": 1300000,
"model_start_us": 1200000,
"model_end_us": 1300000,
"start_resolution_error_us": 0,
"end_resolution_error_us": 0,
"start_frame_index": 2,
"end_frame_index": 3,
"start_second": 0.2,
"end_second": 0.3,
"start_pts_offset_second": 0.2,
"end_pts_offset_second": 0.3,
"technical_block_indexes": [2],
"raw_event_indexes": [2],
"raw_response_sha256": "b",
},
{
"behavior": "放下商品",
"object_change": "商品回到台面",
"start_timestamp": "PTS_US=1400000",
"end_timestamp": "PTS_US=1500000",
"start_us": 1400000,
"end_us": 1500000,
"model_start_us": 1400000,
"model_end_us": 1500000,
"start_resolution_error_us": 0,
"end_resolution_error_us": 0,
"start_frame_index": 4,
"end_frame_index": 5,
"start_second": 0.4,
"end_second": 0.5,
"start_pts_offset_second": 0.4,
"end_pts_offset_second": 0.5,
"technical_block_indexes": [2],
"raw_event_indexes": [3],
"raw_response_sha256": "b",
},
]
merged = operator._merge_adjacent_behaviors(events)
self.assertEqual(len(merged), 2)
self.assertEqual(merged[0]["object_change"], "商品离开货架")
self.assertEqual(merged[0]["end_us"], 1300000)
self.assertEqual(merged[0]["technical_block_indexes"], [1, 2])
self.assertEqual(merged[0]["raw_event_indexes"], [1, 2])
def test_technical_blocks_are_explicitly_not_clip_boundaries(self) -> None:
blocks = operator._technical_blocks(251, 25.0, 10.0, 18)
self.assertEqual(len(blocks), 2)
self.assertTrue(all(block["technicalOnly"] for block in blocks))
self.assertTrue(all("startSecond" not in block and "endSecond" not in block for block in blocks))
self.assertEqual(len(blocks[0]["sampleFrameIndices"]), 18)
def test_model_frames_are_evenly_bounded_and_frame_ids_map_to_exact_pts(self) -> None:
frames = [
{"path": f"frame-{index}.jpg", "frameIndex": index * 10, "timestampUs": 1_000_000 + index * 100_000}
for index in range(18)
]
selected = operator._model_frames(frames)
self.assertEqual([item["plannedFramePosition"] for item in selected], [0, 3, 7, 10, 14, 17])
self.assertEqual([item["modelFrameId"] for item in selected], list(range(6)))
event = operator._normalize_frame_behavior(
{
"behavior": "夹取食品",
"object_change": "食品从加热柜移动到餐盒",
"start_frame_id": 2,
"end_frame_id": 2,
},
selected,
{item["timestampUs"]: item["frameIndex"] for item in frames},
1_000_000,
30.0,
0.5,
)
self.assertEqual((event["model_start_frame_id"], event["model_end_frame_id"]), (2, 2))
self.assertEqual((event["resolved_start_frame_id"], event["resolved_end_frame_id"]), (2, 3))
self.assertEqual(event["start_timestamp"], f"PTS_US={selected[2]['timestampUs']}")
self.assertEqual(event["end_timestamp"], f"PTS_US={selected[3]['timestampUs']}")
self.assertEqual(event["start_second"], (selected[2]["timestampUs"] - 1_000_000) / 2_000_000)
self.assertEqual(event["pts_to_media_scale"], 0.5)
legacy = operator._normalize_model_behavior(
{
"behavior": "夹取食品",
"object_change": "食品移动",
"start_timestamp": f"PTS_US={selected[1]['timestampUs']}",
"end_timestamp": f"PTS_US={selected[2]['timestampUs']}",
},
selected,
{item["timestampUs"]: item["frameIndex"] for item in frames},
1_000_000,
30.0,
1,
0.5,
)
self.assertEqual(legacy["start_second"], (selected[1]["timestampUs"] - 1_000_000) / 2_000_000)
with self.assertRaisesRegex(RuntimeError, "either frame-ID fields"):
operator._normalize_model_behavior(
{"behavior": "bad", "start": 0, "end": 1}, selected, {}, 1_000_000, 30.0, 1, 1.0
)
def test_invalid_behavior_schema_triggers_one_model_retry(self) -> None:
self.assertTrue(
operator._behavior_response_requires_schema_retry(
{"behaviors": [{"behavior": "取出商品", "object_change": "商品离开冷柜"}]}
)
)
self.assertFalse(
operator._behavior_response_requires_schema_retry(
{"behaviors": [{"behavior": "取出商品", "object_change": "商品离开冷柜", "start_frame_id": 0, "end_frame_id": 2}]}
)
)
self.assertFalse(operator._behavior_response_requires_schema_retry({"behaviors": []}))
def test_face_blur_video_pairs_pts_from_orbbec_metadata(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = pathlib.Path(directory)
video = root / "blurred_capture_camera_left_part0001.mp4"
video.write_bytes(b"real-sanitized-video-bytes")
pts = root / "capture_camera_left_part0001_pts.csv"
pts.write_text("timestamp_us\n1\n2\n", encoding="utf-8")
inputs = [
{
"id": "blurred-video-artifact",
"name": video.name,
"path": "bos://testxyz/artifacts/blurred-video.mp4",
},
{
"name": "blurred-manifest.json",
"data": {
"schemaVersion": "droboticflow.video_face_blur.manifest/v1",
"items": [
{
"filename": video.name,
"sanitizedFilename": video.name,
"sanitizedVideoPath": str(video),
"relativePath": "capture_camera_left_part0001.mp4",
"pairedFiles": {"pts": "blurred_capture_camera_left_part0001_pts.csv"},
}
],
},
},
{
"name": "orbbec-normalized-manifest.json",
"data": {
"schemaVersion": "droboticflow.orbbec.normalized/v1",
"datasetType": "orbbec-ego",
"entries": [{"relativePath": pts.name, "path": str(pts), "source": {"path": str(pts)}}],
},
},
]
helpers = {
"payloads_from_inputs": lambda values: [item["data"] for item in values if isinstance(item.get("data"), dict)],
"source_ref_from_artifact": lambda artifact: {"path": artifact["path"]},
"source_path_from_item": lambda item, _name, _output, _archive: pathlib.Path(item.get("path") or item["source"]["path"]),
}
contexts = operator._source_contexts(inputs, root, helpers)
self.assertEqual(len(contexts), 1)
self.assertEqual(contexts[0]["sourcePath"], video)
self.assertEqual(contexts[0]["ptsPath"], pts)
self.assertEqual(contexts[0]["ptsOrigin"], "orbbec-normalized-manifest")
def test_pts_can_be_materialized_from_directory_manifest_inline_bytes(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = pathlib.Path(directory)
video = root / "blurred_capture_camera_left_part0001.mp4"
video.write_bytes(b"real-sanitized-video-bytes")
inputs = [
{"id": "video", "name": video.name, "path": "bos://testxyz/artifacts/blurred.mp4"},
{"name": "blurred-manifest.json", "data": {"schemaVersion": "droboticflow.video_face_blur.manifest/v1", "items": [{"sanitizedFilename": video.name, "sanitizedVideoPath": str(video), "relativePath": "capture_camera_left_part0001.mp4", "pairedFiles": {"pts": "capture_camera_left_part0001_pts.csv"}}]}},
{"name": "orbbec-normalized-manifest.json", "data": {"schemaVersion": "droboticflow.orbbec.normalized/v1", "datasetType": "orbbec-ego", "entries": [{"path": "capture_camera_left_part0001_pts.csv", "inlineText": "timestamp_us\n1000000\n1033333\n"}]}},
]
helpers = {
"payloads_from_inputs": lambda values: [item["data"] for item in values if isinstance(item.get("data"), dict)],
"source_ref_from_artifact": lambda artifact: {"path": artifact["path"]},
"source_path_from_item": lambda *_args: (_ for _ in ()).throw(RuntimeError("BOS must not be used")),
}
contexts = operator._source_contexts(inputs, root, helpers)
self.assertEqual(contexts[0]["ptsOrigin"], "orbbec-normalized-inline-pts")
self.assertEqual(operator._read_pts_csv(contexts[0]["ptsPath"]), [1000000, 1033333])
def test_virtual_clip_keeps_full_source_path_and_ref(self) -> None:
with tempfile.TemporaryDirectory() as directory:
source = pathlib.Path(directory) / "blurred_full.mp4"
source.write_bytes(b"sanitized-video")
source_artifact = {
"id": "face-blur-video",
"name": source.name,
"path": "bos://accepted-bucket/outputs/blurred_full.mp4",
"sizeBytes": source.stat().st_size,
"metadata": {"sha256": "a" * 64, "mimeType": "video/mp4"},
"data": {
"collectionId": "capture-01",
"transformChain": [
{"kind": "opencv-undistort", "applied": True},
{"kind": "face-anonymization", "applied": True},
],
},
"payload": {
"mode": "ref",
"type": "video",
"ref": {"provider": "BOS", "connectionId": "conn-1", "bucket": "accepted-bucket", "key": "outputs/blurred_full.mp4", "kind": "object"},
},
}
captured: dict[str, object] = {}
def build(*args, **kwargs):
captured["args"] = args
captured["kwargs"] = kwargs
return {"name": args[2], "path": args[3], "data": kwargs["data"]}
clip = {
"clipId": "rough-1",
"clipName": "rough-1.mp4",
"startSecond": 1.0,
"endSecond": 2.0,
"sourceDurationSeconds": 25.0,
"behavior": "拿起商品",
"objectChange": "商品离开货架",
}
artifact = operator._virtual_clip_artifact(
"run-1",
"rough",
{"id": "out-1"},
source_artifact,
source,
clip,
{"build_virtual_artifact": build, "source_ref_from_artifact": lambda value: {"path": value["path"]}},
)
self.assertEqual(artifact["path"], source_artifact["path"])
self.assertTrue(artifact["data"]["sourceIsFullVideo"])
self.assertEqual(artifact["data"]["sourceDurationSeconds"], 25.0)
self.assertEqual(artifact["data"]["ref"], source_artifact["payload"]["ref"])
self.assertEqual([item["kind"] for item in artifact["data"]["transformChain"]], ["opencv-undistort", "face-anonymization"])
self.assertEqual(artifact["payload"], source_artifact["payload"])
def test_catalog_has_only_technical_block_parameters_and_expected_ports(self) -> None:
catalog = json.loads((ROOT / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8"))
item = next(value for value in catalog if value.get("id") == "operator-builtin-qwen-behavior-rough-cut")
params = {str(value["key"]) for value in item["hyperparams"]}
self.assertEqual(item["publicationStatus"], "published")
self.assertEqual([value["id"] for value in item["inputs"]], ["videos", "timestamps", "context"])
self.assertEqual([value["id"] for value in item["outputs"]], ["roughCutWindows", "clips", "boundaries", "prelabels", "inferenceSummary"])
self.assertTrue({"TECHNICAL_BLOCK_SECONDS", "FRAMES_PER_TECHNICAL_BLOCK", "PTS_OCR_TOLERANCE_FRAMES"}.issubset(params))
self.assertFalse({"WINDOW_SECONDS", "STRIDE_SECONDS", "OVERLAP_SECONDS", "SEGMENT_DURATION_SECONDS"} & params)
def test_vfr_seek_seconds_use_hardware_pts_not_average_fps(self) -> None:
mapped = operator._normalize_behavior(
{
"behavior": "拿起商品",
"object_change": "商品移动",
"start_timestamp": "PTS_US=1100000",
"end_timestamp": "PTS_US=1500000",
},
{1100000, 1500000},
{1000000: 0, 1100000: 1, 1500000: 2},
1000000,
30.0,
1,
)
self.assertEqual((mapped["start_second"], mapped["end_second"]), (0.1, 0.5))
self.assertNotEqual(mapped["end_second"], round(mapped["end_frame_index"] / 30.0, 6))
def test_runner_registers_dedicated_handler(self) -> None:
runner = (ROOT / "backend/operator-runner/main.py").read_text(encoding="utf-8")
self.assertIn('"qwen_behavior_rough_cut": operator_handler(\'backend.operators.builtin.qwen_behavior_rough_cut.operator\')', runner)
if __name__ == "__main__":
unittest.main()