droboticflow/tests/backend/test_swift_test_utils.py

515 lines
20 KiB
Python

"""Unit tests for ms-swift dataset utilities, taxonomy validation, and causal evaluation golden vectors."""
import json
import pathlib
import pytest
from tests.backend.swift_test_utils import (
CONVENIENCE_STORE_22_TAXONOMY,
check_zero_video_leakage,
extract_source_video_id,
resolve_taxonomy_label,
validate_classification_json_content,
validate_localization_json_content,
validate_swift_jsonl_schema,
)
# ---------------------------------------------------------------------------
# 1. Taxonomy & Alias Resolution Tests
# ---------------------------------------------------------------------------
def test_taxonomy_completeness():
"""Verify official 22 convenience store classes are all defined with IDs 01..22."""
assert len(CONVENIENCE_STORE_22_TAXONOMY) == 22
for i in range(1, 23):
key = f"{i:02d}"
assert key in CONVENIENCE_STORE_22_TAXONOMY
item = CONVENIENCE_STORE_22_TAXONOMY[key]
assert "name" in item and len(item["name"]) > 0
assert "color" in item and item["color"].startswith("#")
assert "aliases" in item and isinstance(item["aliases"], list)
def test_resolve_taxonomy_label_by_id_name_and_alias():
"""Verify alias and ID resolution."""
# By ID
res = resolve_taxonomy_label("05")
assert res == ("05", "蒸点陈列与出餐")
res = resolve_taxonomy_label("5")
assert res == ("05", "蒸点陈列与出餐")
# By exact name
res = resolve_taxonomy_label("热食加工与出餐")
assert res == ("06", "热食加工与出餐")
# By alias
res = resolve_taxonomy_label("烤肠售卖")
assert res == ("06", "热食加工与出餐")
res = resolve_taxonomy_label("便当陈列")
assert res == ("07", "即食商品陈列")
# Unknown
assert resolve_taxonomy_label("非便利店未知动作") is None
# ---------------------------------------------------------------------------
# 2. ms-swift JSONL Schema Validation Tests
# ---------------------------------------------------------------------------
def test_validate_swift_jsonl_schema_happy_path(tmp_path: pathlib.Path):
"""Verify valid ms-swift conversation JSONL passes validation."""
jsonl_file = tmp_path / "valid.jsonl"
data = [
{
"messages": [
{
"role": "user",
"content": "<video>/data/DJI_20260810_0001_D/win_000.mp4</video>\n请识别当前200秒视频中的行为时间区间",
},
{
"role": "assistant",
"content": '{"recognized_labels":["05"],"segments":[{"start":3.5,"end":44.0,"label_id":"05","label":"蒸点陈列与出餐"}]}',
},
],
"videos": ["/data/DJI_20260810_0001_D/win_000.mp4"],
},
{
"messages": [
{
"role": "user",
"content": "<video>/data/DJI_20260810_0002_D/win_000.mp4</video>\n请识别行为",
},
{
"role": "assistant",
"content": '{"recognized_labels":[],"segments":[]}',
},
],
"videos": ["/data/DJI_20260810_0002_D/win_000.mp4"],
},
]
with jsonl_file.open("w", encoding="utf-8") as f:
for row in data:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
result = validate_swift_jsonl_schema(jsonl_file)
assert result["valid"] is True
assert result["total_lines"] == 2
assert result["valid_lines"] == 2
assert len(result["errors"]) == 0
assert len(result["videos"]) == 2
def test_validate_swift_jsonl_schema_invalid_cases(tmp_path: pathlib.Path):
"""Verify invalid JSONL formats trigger validation errors."""
# 1. Non-existent file
res = validate_swift_jsonl_schema(tmp_path / "non_existent.jsonl")
assert res["valid"] is False
assert "File not found" in res["errors"][0]
# 2. Corrupt JSON syntax
bad_json = tmp_path / "bad_syntax.jsonl"
bad_json.write_text("{\"messages\": [INVALID JSON]}\n", encoding="utf-8")
res = validate_swift_jsonl_schema(bad_json)
assert res["valid"] is False
assert any("Invalid JSON syntax" in e for e in res["errors"])
# 3. Missing assistant message
missing_assistant = tmp_path / "missing_assistant.jsonl"
missing_assistant.write_text(
json.dumps({"messages": [{"role": "user", "content": "hello"}]}) + "\n",
encoding="utf-8",
)
res = validate_swift_jsonl_schema(missing_assistant)
assert res["valid"] is False
assert any("at least 2 messages" in e for e in res["errors"])
# ---------------------------------------------------------------------------
# 3. Source Video Extraction and Zero Leakage Tests
# ---------------------------------------------------------------------------
def test_extract_source_video_id():
"""Verify source video identifier extraction across path conventions."""
assert (
extract_source_video_id("/zerogap_nas/delivery/d/DJI_20260810120000_0001_D/windows/train/win_000.mp4")
== "DJI_20260810120000_0001_D"
)
assert (
extract_source_video_id("/data/DJI_20260810_0002_D.MP4")
== "DJI_20260810_0002_D"
)
assert (
extract_source_video_id("Orbbec_Ego_AZER76400GG_20260810_120000/Orbbec_Ego_AZER76400GG_20260810_120000_camera_left_part0001.mp4")
== "Orbbec_Ego_AZER76400GG_20260810_120000"
)
def test_check_zero_video_leakage_clean_splits(tmp_path: pathlib.Path):
"""Verify zero leakage passes when splits have distinct video sources."""
train_file = tmp_path / "train.jsonl"
val_file = tmp_path / "val.jsonl"
test_file = tmp_path / "test.jsonl"
train_file.write_text(
json.dumps({
"messages": [{"role": "user", "content": "win1"}, {"role": "assistant", "content": "{}"}],
"videos": ["/data/DJI_20260810_0001_D/win_000.mp4", "/data/DJI_20260810_0002_D/win_000.mp4"],
}) + "\n",
encoding="utf-8",
)
val_file.write_text(
json.dumps({
"messages": [{"role": "user", "content": "win2"}, {"role": "assistant", "content": "{}"}],
"videos": ["/data/DJI_20260810_0003_D/win_000.mp4"],
}) + "\n",
encoding="utf-8",
)
test_file.write_text(
json.dumps({
"messages": [{"role": "user", "content": "win3"}, {"role": "assistant", "content": "{}"}],
"videos": ["/data/DJI_20260810_0004_D/win_000.mp4"],
}) + "\n",
encoding="utf-8",
)
leakage_report = check_zero_video_leakage(train_file, val_file, test_file)
assert leakage_report["zero_leakage"] is True
assert leakage_report["leakage_detected"] is False
assert len(leakage_report["overlapping_sources"]) == 0
assert leakage_report["train_sources_count"] == 2
assert leakage_report["val_sources_count"] == 1
assert leakage_report["test_sources_count"] == 1
def test_check_zero_video_leakage_detects_leakage(tmp_path: pathlib.Path):
"""Verify leakage is detected when the same video appears in train and val."""
train_file = tmp_path / "train_leaked.jsonl"
val_file = tmp_path / "val_leaked.jsonl"
train_file.write_text(
json.dumps({
"messages": [{"role": "user", "content": "win1"}, {"role": "assistant", "content": "{}"}],
"videos": ["/data/DJI_20260810_0001_D/win_000.mp4"],
}) + "\n",
encoding="utf-8",
)
val_file.write_text(
json.dumps({
"messages": [{"role": "user", "content": "win2"}, {"role": "assistant", "content": "{}"}],
"videos": ["/data/DJI_20260810_0001_D/win_001.mp4"], # Same source video!
}) + "\n",
encoding="utf-8",
)
leakage_report = check_zero_video_leakage(train_file, val_file)
assert leakage_report["zero_leakage"] is False
assert leakage_report["leakage_detected"] is True
assert "DJI_20260810_0001_D" in leakage_report["overlapping_sources"]
assert "DJI_20260810_0001_D" in leakage_report["overlap_details"]["train_val"]
# ---------------------------------------------------------------------------
# 4. Localization JSON Content Validation Tests
# ---------------------------------------------------------------------------
def test_validate_localization_json_content_valid():
"""Verify valid localization JSON passes validation."""
content = json.dumps({
"recognized_labels": ["05", "06"],
"segments": [
{"start": 3.5, "end": 44.0, "label_id": "05", "label": "蒸点陈列与出餐"},
{"start": 34.3, "end": 63.3, "label_id": "06", "label": "热食加工与出餐"},
],
})
res = validate_localization_json_content(content, max_duration=200.0)
assert res["valid"] is True
assert len(res["errors"]) == 0
assert res["recognized_labels"] == ["05", "06"]
assert res["segment_count"] == 2
def test_validate_localization_json_content_empty_valid():
"""Verify empty localization JSON is valid."""
content = json.dumps({"recognized_labels": [], "segments": []})
res = validate_localization_json_content(content, max_duration=200.0)
assert res["valid"] is True
assert res["segment_count"] == 0
def test_validate_localization_json_content_invalid_cases():
"""Verify various localization errors are caught."""
# 1. start >= end
bad_time = json.dumps({
"recognized_labels": ["01"],
"segments": [{"start": 10.0, "end": 5.0, "label_id": "01", "label": "鲜食处理"}],
})
res = validate_localization_json_content(bad_time)
assert res["valid"] is False
assert any("greater than 'start'" in e for e in res["errors"])
# 2. Exceeds max_duration
out_of_bounds = json.dumps({
"recognized_labels": ["01"],
"segments": [{"start": 10.0, "end": 250.0, "label_id": "01", "label": "鲜食处理"}],
})
res = validate_localization_json_content(out_of_bounds, max_duration=200.0)
assert res["valid"] is False
assert any("exceeds window max_duration" in e for e in res["errors"])
# 3. Unknown label_id
unknown_id = json.dumps({
"recognized_labels": ["99"],
"segments": [{"start": 0.0, "end": 10.0, "label_id": "99", "label": "未知动作"}],
})
res = validate_localization_json_content(unknown_id)
assert res["valid"] is False
assert any("Unknown label_id '99'" in e for e in res["errors"])
# 4. Mismatch between recognized_labels and segments
mismatched = json.dumps({
"recognized_labels": ["01", "02"],
"segments": [{"start": 0.0, "end": 10.0, "label_id": "01", "label": "鲜食处理"}],
})
res = validate_localization_json_content(mismatched)
assert res["valid"] is False
assert any("does not match unique segment label_ids" in e for e in res["errors"])
# ---------------------------------------------------------------------------
# 5. Classification JSON Content Validation Tests
# ---------------------------------------------------------------------------
def test_validate_classification_json_content_valid():
"""Verify valid classification JSON passes validation."""
content = json.dumps({
"labels": [
{"label_id": "05", "label": "蒸点陈列与出餐"},
{"label_id": "06", "label": "热食加工与出餐"},
]
})
res = validate_classification_json_content(content)
assert res["valid"] is True
assert res["label_count"] == 2
def test_validate_classification_json_content_empty_valid():
"""Verify empty classification labels list is valid."""
content = json.dumps({"labels": []})
res = validate_classification_json_content(content)
assert res["valid"] is True
assert res["label_count"] == 0
def test_validate_classification_json_content_duplicates_and_invalid():
"""Verify duplicate label_ids or invalid structure fail validation."""
# Duplicate label_id
duplicate = json.dumps({
"labels": [
{"label_id": "05", "label": "蒸点陈列与出餐"},
{"label_id": "05", "label": "蒸点陈列与出餐"},
]
})
res = validate_classification_json_content(duplicate)
assert res["valid"] is False
assert any("Duplicate label_id '05'" in e for e in res["errors"])
# ---------------------------------------------------------------------------
# 6. Causal Evaluation Golden Vectors Mathematical Verification
# ---------------------------------------------------------------------------
def compute_temporal_iou(interval_a: list[float], interval_b: list[float]) -> tuple[float, float, float]:
"""Compute (intersection, union, iou) for two intervals [start, end]."""
s_a, e_a = interval_a
s_b, e_b = interval_b
inter = max(0.0, min(e_a, e_b) - max(s_a, s_b))
union = (e_a - s_a) + (e_b - s_b) - inter
iou = inter / union if union > 0 else 0.0
return inter, union, iou
def merge_temporal_windows(
windows_with_segments: list[dict],
max_gap: float = 2.0,
) -> list[dict]:
"""Merge temporal segments across windows based on absolute time and matching label_id."""
abs_segments = []
for win in windows_with_segments:
w_start = win["start_abs"]
for seg in win["segments"]:
abs_segments.append({
"start_abs": w_start + seg["start_rel"],
"end_abs": w_start + seg["end_rel"],
"label_id": seg["label_id"],
"label": seg["label"],
})
# Group by label_id and sort by start_abs
by_label: dict[str, list[dict]] = {}
for seg in abs_segments:
by_label.setdefault(seg["label_id"], []).append(seg)
merged: list[dict] = []
for lid in sorted(by_label.keys()):
segs = sorted(by_label[lid], key=lambda x: x["start_abs"])
cur = segs[0]
for nxt in segs[1:]:
# Check overlap or gap <= max_gap
if nxt["start_abs"] <= cur["end_abs"] + max_gap:
cur["end_abs"] = max(cur["end_abs"], nxt["end_abs"])
else:
merged.append(cur)
cur = nxt
merged.append(cur)
return sorted(merged, key=lambda x: (x["start_abs"], x["label_id"]))
def test_golden_vectors_temporal_iou():
"""Verify all temporal IoU vectors against mathematical ground truth."""
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
with gv_path.open("r", encoding="utf-8") as f:
golden = json.load(f)
for vec in golden["temporal_iou_vectors"]:
inter, union, iou = compute_temporal_iou(vec["gt"], vec["pred"])
assert pytest.approx(inter, 1e-6) == vec["intersection"], f"Failed on {vec['name']} inter"
assert pytest.approx(union, 1e-6) == vec["union"], f"Failed on {vec['name']} union"
assert pytest.approx(iou, 1e-6) == vec["iou"], f"Failed on {vec['name']} iou"
assert (iou >= 0.3) == vec["pass_at_0_3"], f"Failed on {vec['name']} @0.3"
assert (iou >= 0.5) == vec["pass_at_0_5"], f"Failed on {vec['name']} @0.5"
assert (iou >= 0.7) == vec["pass_at_0_7"], f"Failed on {vec['name']} @0.7"
def test_golden_vectors_boundary_mae():
"""Verify boundary MAE calculations on golden vectors."""
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
with gv_path.open("r", encoding="utf-8") as f:
golden = json.load(f)
mae_data = golden["boundary_mae_vectors"]
sp = mae_data["single_pair"]
inter, union, iou = compute_temporal_iou(sp["gt"], sp["pred"])
assert iou >= 0.5
s_err = abs(sp["pred"][0] - sp["gt"][0])
e_err = abs(sp["pred"][1] - sp["gt"][1])
pair_mae = (s_err + e_err) / 2.0
assert pytest.approx(pair_mae, 1e-6) == sp["pair_mae"]
mp = mae_data["multi_pair_evaluation"]
total_mae = 0.0
for p in mp["pairs"]:
p_s_err = abs(p["pred"][0] - p["gt"][0])
p_e_err = abs(p["pred"][1] - p["gt"][1])
p_mae = (p_s_err + p_e_err) / 2.0
assert pytest.approx(p_mae, 1e-6) == p["pair_mae"]
total_mae += p_mae
avg_mae = total_mae / len(mp["pairs"])
assert pytest.approx(avg_mae, 1e-6) == mp["boundary_mae_seconds_at_iou_0_5"]
def test_golden_vectors_f1_multilabel():
"""Verify Micro-F1 and Macro-F1 golden vectors mathematically."""
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
with gv_path.open("r", encoding="utf-8") as f:
golden = json.load(f)
f1_data = golden["f1_multi_label_vectors"]
samples = f1_data["samples"]
tp_total = 0
fp_total = 0
fn_total = 0
exact_matches = 0
per_class_counts: dict[str, dict[str, int]] = {
lid: {"tp": 0, "fp": 0, "fn": 0} for lid in f1_data["taxonomy_ids"]
}
for s in samples:
gt_set = set(s["gt_labels"])
pred_set = set(s["pred_labels"])
if gt_set == pred_set:
exact_matches += 1
for lid in f1_data["taxonomy_ids"]:
in_gt = lid in gt_set
in_pred = lid in pred_set
if in_gt and in_pred:
per_class_counts[lid]["tp"] += 1
tp_total += 1
elif not in_gt and in_pred:
per_class_counts[lid]["fp"] += 1
fp_total += 1
elif in_gt and not in_pred:
per_class_counts[lid]["fn"] += 1
fn_total += 1
# Verify pooled counts
assert tp_total == f1_data["pooled_counts"]["tp"]
assert fp_total == f1_data["pooled_counts"]["fp"]
assert fn_total == f1_data["pooled_counts"]["fn"]
# Micro F1
p_micro = tp_total / (tp_total + fp_total)
r_micro = tp_total / (tp_total + fn_total)
f1_micro = 2 * p_micro * r_micro / (p_micro + r_micro)
metrics = f1_data["metrics"]
assert pytest.approx(p_micro, 1e-6) == metrics["precision_micro"]
assert pytest.approx(r_micro, 1e-6) == metrics["recall_micro"]
assert pytest.approx(f1_micro, 1e-6) == metrics["f1_micro"]
assert pytest.approx(exact_matches / len(samples), 1e-6) == metrics["exact_match_accuracy"]
# Per-class & Macro F1
class_f1s = []
gold_f1s = []
for lid, counts in per_class_counts.items():
tp = counts["tp"]
fp = counts["fp"]
fn = counts["fn"]
p = tp / (tp + fp) if (tp + fp) > 0 else 0.0
r = tp / (tp + fn) if (tp + fn) > 0 else 0.0
f1 = (2 * p * r / (p + r)) if (p + r) > 0 else 0.0
class_f1s.append(f1)
if (tp + fn) > 0:
gold_f1s.append(f1)
if lid in f1_data["per_class_f1"]:
expected = f1_data["per_class_f1"][lid]
assert counts["tp"] == expected["tp"]
assert counts["fp"] == expected["fp"]
assert counts["fn"] == expected["fn"]
assert pytest.approx(f1, 1e-6) == expected["f1"]
macro_f1_all = sum(class_f1s) / len(class_f1s)
macro_f1_gold = sum(gold_f1s) / len(gold_f1s)
assert pytest.approx(macro_f1_all, 1e-6) == metrics["macro_f1_all_22_classes"]
assert pytest.approx(macro_f1_gold, 1e-6) == metrics["macro_f1_gold_classes"]
def test_golden_vectors_window_merging():
"""Verify window merging algorithm against all golden test cases."""
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
with gv_path.open("r", encoding="utf-8") as f:
golden = json.load(f)
merging_data = golden["window_merging_vectors"]
max_gap = merging_data["max_gap_seconds"]
for case in merging_data["cases"]:
windows = case["windows"]
merged = merge_temporal_windows(windows, max_gap=max_gap)
expected = case["expected_merged_segments"]
assert len(merged) == len(expected), f"Case {case['case_id']} length mismatch"
for actual_seg, exp_seg in zip(merged, expected):
assert pytest.approx(actual_seg["start_abs"], 1e-6) == exp_seg["start_abs"]
assert pytest.approx(actual_seg["end_abs"], 1e-6) == exp_seg["end_abs"]
assert actual_seg["label_id"] == exp_seg["label_id"]
assert actual_seg["label"] == exp_seg["label"]