515 lines
20 KiB
Python
515 lines
20 KiB
Python
"""Unit tests for ms-swift dataset utilities, taxonomy validation, and causal evaluation golden vectors."""
|
|
|
|
import json
|
|
import pathlib
|
|
import pytest
|
|
|
|
from tests.backend.swift_test_utils import (
|
|
CONVENIENCE_STORE_22_TAXONOMY,
|
|
check_zero_video_leakage,
|
|
extract_source_video_id,
|
|
resolve_taxonomy_label,
|
|
validate_classification_json_content,
|
|
validate_localization_json_content,
|
|
validate_swift_jsonl_schema,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. Taxonomy & Alias Resolution Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_taxonomy_completeness():
|
|
"""Verify official 22 convenience store classes are all defined with IDs 01..22."""
|
|
assert len(CONVENIENCE_STORE_22_TAXONOMY) == 22
|
|
for i in range(1, 23):
|
|
key = f"{i:02d}"
|
|
assert key in CONVENIENCE_STORE_22_TAXONOMY
|
|
item = CONVENIENCE_STORE_22_TAXONOMY[key]
|
|
assert "name" in item and len(item["name"]) > 0
|
|
assert "color" in item and item["color"].startswith("#")
|
|
assert "aliases" in item and isinstance(item["aliases"], list)
|
|
|
|
|
|
def test_resolve_taxonomy_label_by_id_name_and_alias():
|
|
"""Verify alias and ID resolution."""
|
|
# By ID
|
|
res = resolve_taxonomy_label("05")
|
|
assert res == ("05", "蒸点陈列与出餐")
|
|
|
|
res = resolve_taxonomy_label("5")
|
|
assert res == ("05", "蒸点陈列与出餐")
|
|
|
|
# By exact name
|
|
res = resolve_taxonomy_label("热食加工与出餐")
|
|
assert res == ("06", "热食加工与出餐")
|
|
|
|
# By alias
|
|
res = resolve_taxonomy_label("烤肠售卖")
|
|
assert res == ("06", "热食加工与出餐")
|
|
|
|
res = resolve_taxonomy_label("便当陈列")
|
|
assert res == ("07", "即食商品陈列")
|
|
|
|
# Unknown
|
|
assert resolve_taxonomy_label("非便利店未知动作") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. ms-swift JSONL Schema Validation Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_validate_swift_jsonl_schema_happy_path(tmp_path: pathlib.Path):
|
|
"""Verify valid ms-swift conversation JSONL passes validation."""
|
|
jsonl_file = tmp_path / "valid.jsonl"
|
|
data = [
|
|
{
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "<video>/data/DJI_20260810_0001_D/win_000.mp4</video>\n请识别当前200秒视频中的行为时间区间",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": '{"recognized_labels":["05"],"segments":[{"start":3.5,"end":44.0,"label_id":"05","label":"蒸点陈列与出餐"}]}',
|
|
},
|
|
],
|
|
"videos": ["/data/DJI_20260810_0001_D/win_000.mp4"],
|
|
},
|
|
{
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "<video>/data/DJI_20260810_0002_D/win_000.mp4</video>\n请识别行为",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": '{"recognized_labels":[],"segments":[]}',
|
|
},
|
|
],
|
|
"videos": ["/data/DJI_20260810_0002_D/win_000.mp4"],
|
|
},
|
|
]
|
|
with jsonl_file.open("w", encoding="utf-8") as f:
|
|
for row in data:
|
|
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
|
|
result = validate_swift_jsonl_schema(jsonl_file)
|
|
assert result["valid"] is True
|
|
assert result["total_lines"] == 2
|
|
assert result["valid_lines"] == 2
|
|
assert len(result["errors"]) == 0
|
|
assert len(result["videos"]) == 2
|
|
|
|
|
|
def test_validate_swift_jsonl_schema_invalid_cases(tmp_path: pathlib.Path):
|
|
"""Verify invalid JSONL formats trigger validation errors."""
|
|
# 1. Non-existent file
|
|
res = validate_swift_jsonl_schema(tmp_path / "non_existent.jsonl")
|
|
assert res["valid"] is False
|
|
assert "File not found" in res["errors"][0]
|
|
|
|
# 2. Corrupt JSON syntax
|
|
bad_json = tmp_path / "bad_syntax.jsonl"
|
|
bad_json.write_text("{\"messages\": [INVALID JSON]}\n", encoding="utf-8")
|
|
res = validate_swift_jsonl_schema(bad_json)
|
|
assert res["valid"] is False
|
|
assert any("Invalid JSON syntax" in e for e in res["errors"])
|
|
|
|
# 3. Missing assistant message
|
|
missing_assistant = tmp_path / "missing_assistant.jsonl"
|
|
missing_assistant.write_text(
|
|
json.dumps({"messages": [{"role": "user", "content": "hello"}]}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
res = validate_swift_jsonl_schema(missing_assistant)
|
|
assert res["valid"] is False
|
|
assert any("at least 2 messages" in e for e in res["errors"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. Source Video Extraction and Zero Leakage Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_extract_source_video_id():
|
|
"""Verify source video identifier extraction across path conventions."""
|
|
assert (
|
|
extract_source_video_id("/zerogap_nas/delivery/d/DJI_20260810120000_0001_D/windows/train/win_000.mp4")
|
|
== "DJI_20260810120000_0001_D"
|
|
)
|
|
assert (
|
|
extract_source_video_id("/data/DJI_20260810_0002_D.MP4")
|
|
== "DJI_20260810_0002_D"
|
|
)
|
|
assert (
|
|
extract_source_video_id("Orbbec_Ego_AZER76400GG_20260810_120000/Orbbec_Ego_AZER76400GG_20260810_120000_camera_left_part0001.mp4")
|
|
== "Orbbec_Ego_AZER76400GG_20260810_120000"
|
|
)
|
|
|
|
|
|
def test_check_zero_video_leakage_clean_splits(tmp_path: pathlib.Path):
|
|
"""Verify zero leakage passes when splits have distinct video sources."""
|
|
train_file = tmp_path / "train.jsonl"
|
|
val_file = tmp_path / "val.jsonl"
|
|
test_file = tmp_path / "test.jsonl"
|
|
|
|
train_file.write_text(
|
|
json.dumps({
|
|
"messages": [{"role": "user", "content": "win1"}, {"role": "assistant", "content": "{}"}],
|
|
"videos": ["/data/DJI_20260810_0001_D/win_000.mp4", "/data/DJI_20260810_0002_D/win_000.mp4"],
|
|
}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
val_file.write_text(
|
|
json.dumps({
|
|
"messages": [{"role": "user", "content": "win2"}, {"role": "assistant", "content": "{}"}],
|
|
"videos": ["/data/DJI_20260810_0003_D/win_000.mp4"],
|
|
}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
test_file.write_text(
|
|
json.dumps({
|
|
"messages": [{"role": "user", "content": "win3"}, {"role": "assistant", "content": "{}"}],
|
|
"videos": ["/data/DJI_20260810_0004_D/win_000.mp4"],
|
|
}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
leakage_report = check_zero_video_leakage(train_file, val_file, test_file)
|
|
assert leakage_report["zero_leakage"] is True
|
|
assert leakage_report["leakage_detected"] is False
|
|
assert len(leakage_report["overlapping_sources"]) == 0
|
|
assert leakage_report["train_sources_count"] == 2
|
|
assert leakage_report["val_sources_count"] == 1
|
|
assert leakage_report["test_sources_count"] == 1
|
|
|
|
|
|
def test_check_zero_video_leakage_detects_leakage(tmp_path: pathlib.Path):
|
|
"""Verify leakage is detected when the same video appears in train and val."""
|
|
train_file = tmp_path / "train_leaked.jsonl"
|
|
val_file = tmp_path / "val_leaked.jsonl"
|
|
|
|
train_file.write_text(
|
|
json.dumps({
|
|
"messages": [{"role": "user", "content": "win1"}, {"role": "assistant", "content": "{}"}],
|
|
"videos": ["/data/DJI_20260810_0001_D/win_000.mp4"],
|
|
}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
val_file.write_text(
|
|
json.dumps({
|
|
"messages": [{"role": "user", "content": "win2"}, {"role": "assistant", "content": "{}"}],
|
|
"videos": ["/data/DJI_20260810_0001_D/win_001.mp4"], # Same source video!
|
|
}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
leakage_report = check_zero_video_leakage(train_file, val_file)
|
|
assert leakage_report["zero_leakage"] is False
|
|
assert leakage_report["leakage_detected"] is True
|
|
assert "DJI_20260810_0001_D" in leakage_report["overlapping_sources"]
|
|
assert "DJI_20260810_0001_D" in leakage_report["overlap_details"]["train_val"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. Localization JSON Content Validation Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_validate_localization_json_content_valid():
|
|
"""Verify valid localization JSON passes validation."""
|
|
content = json.dumps({
|
|
"recognized_labels": ["05", "06"],
|
|
"segments": [
|
|
{"start": 3.5, "end": 44.0, "label_id": "05", "label": "蒸点陈列与出餐"},
|
|
{"start": 34.3, "end": 63.3, "label_id": "06", "label": "热食加工与出餐"},
|
|
],
|
|
})
|
|
res = validate_localization_json_content(content, max_duration=200.0)
|
|
assert res["valid"] is True
|
|
assert len(res["errors"]) == 0
|
|
assert res["recognized_labels"] == ["05", "06"]
|
|
assert res["segment_count"] == 2
|
|
|
|
|
|
def test_validate_localization_json_content_empty_valid():
|
|
"""Verify empty localization JSON is valid."""
|
|
content = json.dumps({"recognized_labels": [], "segments": []})
|
|
res = validate_localization_json_content(content, max_duration=200.0)
|
|
assert res["valid"] is True
|
|
assert res["segment_count"] == 0
|
|
|
|
|
|
def test_validate_localization_json_content_invalid_cases():
|
|
"""Verify various localization errors are caught."""
|
|
# 1. start >= end
|
|
bad_time = json.dumps({
|
|
"recognized_labels": ["01"],
|
|
"segments": [{"start": 10.0, "end": 5.0, "label_id": "01", "label": "鲜食处理"}],
|
|
})
|
|
res = validate_localization_json_content(bad_time)
|
|
assert res["valid"] is False
|
|
assert any("greater than 'start'" in e for e in res["errors"])
|
|
|
|
# 2. Exceeds max_duration
|
|
out_of_bounds = json.dumps({
|
|
"recognized_labels": ["01"],
|
|
"segments": [{"start": 10.0, "end": 250.0, "label_id": "01", "label": "鲜食处理"}],
|
|
})
|
|
res = validate_localization_json_content(out_of_bounds, max_duration=200.0)
|
|
assert res["valid"] is False
|
|
assert any("exceeds window max_duration" in e for e in res["errors"])
|
|
|
|
# 3. Unknown label_id
|
|
unknown_id = json.dumps({
|
|
"recognized_labels": ["99"],
|
|
"segments": [{"start": 0.0, "end": 10.0, "label_id": "99", "label": "未知动作"}],
|
|
})
|
|
res = validate_localization_json_content(unknown_id)
|
|
assert res["valid"] is False
|
|
assert any("Unknown label_id '99'" in e for e in res["errors"])
|
|
|
|
# 4. Mismatch between recognized_labels and segments
|
|
mismatched = json.dumps({
|
|
"recognized_labels": ["01", "02"],
|
|
"segments": [{"start": 0.0, "end": 10.0, "label_id": "01", "label": "鲜食处理"}],
|
|
})
|
|
res = validate_localization_json_content(mismatched)
|
|
assert res["valid"] is False
|
|
assert any("does not match unique segment label_ids" in e for e in res["errors"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. Classification JSON Content Validation Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_validate_classification_json_content_valid():
|
|
"""Verify valid classification JSON passes validation."""
|
|
content = json.dumps({
|
|
"labels": [
|
|
{"label_id": "05", "label": "蒸点陈列与出餐"},
|
|
{"label_id": "06", "label": "热食加工与出餐"},
|
|
]
|
|
})
|
|
res = validate_classification_json_content(content)
|
|
assert res["valid"] is True
|
|
assert res["label_count"] == 2
|
|
|
|
|
|
def test_validate_classification_json_content_empty_valid():
|
|
"""Verify empty classification labels list is valid."""
|
|
content = json.dumps({"labels": []})
|
|
res = validate_classification_json_content(content)
|
|
assert res["valid"] is True
|
|
assert res["label_count"] == 0
|
|
|
|
|
|
def test_validate_classification_json_content_duplicates_and_invalid():
|
|
"""Verify duplicate label_ids or invalid structure fail validation."""
|
|
# Duplicate label_id
|
|
duplicate = json.dumps({
|
|
"labels": [
|
|
{"label_id": "05", "label": "蒸点陈列与出餐"},
|
|
{"label_id": "05", "label": "蒸点陈列与出餐"},
|
|
]
|
|
})
|
|
res = validate_classification_json_content(duplicate)
|
|
assert res["valid"] is False
|
|
assert any("Duplicate label_id '05'" in e for e in res["errors"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. Causal Evaluation Golden Vectors Mathematical Verification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def compute_temporal_iou(interval_a: list[float], interval_b: list[float]) -> tuple[float, float, float]:
|
|
"""Compute (intersection, union, iou) for two intervals [start, end]."""
|
|
s_a, e_a = interval_a
|
|
s_b, e_b = interval_b
|
|
inter = max(0.0, min(e_a, e_b) - max(s_a, s_b))
|
|
union = (e_a - s_a) + (e_b - s_b) - inter
|
|
iou = inter / union if union > 0 else 0.0
|
|
return inter, union, iou
|
|
|
|
|
|
def merge_temporal_windows(
|
|
windows_with_segments: list[dict],
|
|
max_gap: float = 2.0,
|
|
) -> list[dict]:
|
|
"""Merge temporal segments across windows based on absolute time and matching label_id."""
|
|
abs_segments = []
|
|
for win in windows_with_segments:
|
|
w_start = win["start_abs"]
|
|
for seg in win["segments"]:
|
|
abs_segments.append({
|
|
"start_abs": w_start + seg["start_rel"],
|
|
"end_abs": w_start + seg["end_rel"],
|
|
"label_id": seg["label_id"],
|
|
"label": seg["label"],
|
|
})
|
|
|
|
# Group by label_id and sort by start_abs
|
|
by_label: dict[str, list[dict]] = {}
|
|
for seg in abs_segments:
|
|
by_label.setdefault(seg["label_id"], []).append(seg)
|
|
|
|
merged: list[dict] = []
|
|
for lid in sorted(by_label.keys()):
|
|
segs = sorted(by_label[lid], key=lambda x: x["start_abs"])
|
|
cur = segs[0]
|
|
for nxt in segs[1:]:
|
|
# Check overlap or gap <= max_gap
|
|
if nxt["start_abs"] <= cur["end_abs"] + max_gap:
|
|
cur["end_abs"] = max(cur["end_abs"], nxt["end_abs"])
|
|
else:
|
|
merged.append(cur)
|
|
cur = nxt
|
|
merged.append(cur)
|
|
|
|
return sorted(merged, key=lambda x: (x["start_abs"], x["label_id"]))
|
|
|
|
|
|
def test_golden_vectors_temporal_iou():
|
|
"""Verify all temporal IoU vectors against mathematical ground truth."""
|
|
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
|
|
with gv_path.open("r", encoding="utf-8") as f:
|
|
golden = json.load(f)
|
|
|
|
for vec in golden["temporal_iou_vectors"]:
|
|
inter, union, iou = compute_temporal_iou(vec["gt"], vec["pred"])
|
|
assert pytest.approx(inter, 1e-6) == vec["intersection"], f"Failed on {vec['name']} inter"
|
|
assert pytest.approx(union, 1e-6) == vec["union"], f"Failed on {vec['name']} union"
|
|
assert pytest.approx(iou, 1e-6) == vec["iou"], f"Failed on {vec['name']} iou"
|
|
|
|
assert (iou >= 0.3) == vec["pass_at_0_3"], f"Failed on {vec['name']} @0.3"
|
|
assert (iou >= 0.5) == vec["pass_at_0_5"], f"Failed on {vec['name']} @0.5"
|
|
assert (iou >= 0.7) == vec["pass_at_0_7"], f"Failed on {vec['name']} @0.7"
|
|
|
|
|
|
def test_golden_vectors_boundary_mae():
|
|
"""Verify boundary MAE calculations on golden vectors."""
|
|
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
|
|
with gv_path.open("r", encoding="utf-8") as f:
|
|
golden = json.load(f)
|
|
|
|
mae_data = golden["boundary_mae_vectors"]
|
|
sp = mae_data["single_pair"]
|
|
inter, union, iou = compute_temporal_iou(sp["gt"], sp["pred"])
|
|
assert iou >= 0.5
|
|
s_err = abs(sp["pred"][0] - sp["gt"][0])
|
|
e_err = abs(sp["pred"][1] - sp["gt"][1])
|
|
pair_mae = (s_err + e_err) / 2.0
|
|
assert pytest.approx(pair_mae, 1e-6) == sp["pair_mae"]
|
|
|
|
mp = mae_data["multi_pair_evaluation"]
|
|
total_mae = 0.0
|
|
for p in mp["pairs"]:
|
|
p_s_err = abs(p["pred"][0] - p["gt"][0])
|
|
p_e_err = abs(p["pred"][1] - p["gt"][1])
|
|
p_mae = (p_s_err + p_e_err) / 2.0
|
|
assert pytest.approx(p_mae, 1e-6) == p["pair_mae"]
|
|
total_mae += p_mae
|
|
avg_mae = total_mae / len(mp["pairs"])
|
|
assert pytest.approx(avg_mae, 1e-6) == mp["boundary_mae_seconds_at_iou_0_5"]
|
|
|
|
|
|
def test_golden_vectors_f1_multilabel():
|
|
"""Verify Micro-F1 and Macro-F1 golden vectors mathematically."""
|
|
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
|
|
with gv_path.open("r", encoding="utf-8") as f:
|
|
golden = json.load(f)
|
|
|
|
f1_data = golden["f1_multi_label_vectors"]
|
|
samples = f1_data["samples"]
|
|
|
|
tp_total = 0
|
|
fp_total = 0
|
|
fn_total = 0
|
|
exact_matches = 0
|
|
|
|
per_class_counts: dict[str, dict[str, int]] = {
|
|
lid: {"tp": 0, "fp": 0, "fn": 0} for lid in f1_data["taxonomy_ids"]
|
|
}
|
|
|
|
for s in samples:
|
|
gt_set = set(s["gt_labels"])
|
|
pred_set = set(s["pred_labels"])
|
|
if gt_set == pred_set:
|
|
exact_matches += 1
|
|
|
|
for lid in f1_data["taxonomy_ids"]:
|
|
in_gt = lid in gt_set
|
|
in_pred = lid in pred_set
|
|
if in_gt and in_pred:
|
|
per_class_counts[lid]["tp"] += 1
|
|
tp_total += 1
|
|
elif not in_gt and in_pred:
|
|
per_class_counts[lid]["fp"] += 1
|
|
fp_total += 1
|
|
elif in_gt and not in_pred:
|
|
per_class_counts[lid]["fn"] += 1
|
|
fn_total += 1
|
|
|
|
# Verify pooled counts
|
|
assert tp_total == f1_data["pooled_counts"]["tp"]
|
|
assert fp_total == f1_data["pooled_counts"]["fp"]
|
|
assert fn_total == f1_data["pooled_counts"]["fn"]
|
|
|
|
# Micro F1
|
|
p_micro = tp_total / (tp_total + fp_total)
|
|
r_micro = tp_total / (tp_total + fn_total)
|
|
f1_micro = 2 * p_micro * r_micro / (p_micro + r_micro)
|
|
|
|
metrics = f1_data["metrics"]
|
|
assert pytest.approx(p_micro, 1e-6) == metrics["precision_micro"]
|
|
assert pytest.approx(r_micro, 1e-6) == metrics["recall_micro"]
|
|
assert pytest.approx(f1_micro, 1e-6) == metrics["f1_micro"]
|
|
assert pytest.approx(exact_matches / len(samples), 1e-6) == metrics["exact_match_accuracy"]
|
|
|
|
# Per-class & Macro F1
|
|
class_f1s = []
|
|
gold_f1s = []
|
|
for lid, counts in per_class_counts.items():
|
|
tp = counts["tp"]
|
|
fp = counts["fp"]
|
|
fn = counts["fn"]
|
|
p = tp / (tp + fp) if (tp + fp) > 0 else 0.0
|
|
r = tp / (tp + fn) if (tp + fn) > 0 else 0.0
|
|
f1 = (2 * p * r / (p + r)) if (p + r) > 0 else 0.0
|
|
class_f1s.append(f1)
|
|
if (tp + fn) > 0:
|
|
gold_f1s.append(f1)
|
|
|
|
if lid in f1_data["per_class_f1"]:
|
|
expected = f1_data["per_class_f1"][lid]
|
|
assert counts["tp"] == expected["tp"]
|
|
assert counts["fp"] == expected["fp"]
|
|
assert counts["fn"] == expected["fn"]
|
|
assert pytest.approx(f1, 1e-6) == expected["f1"]
|
|
|
|
macro_f1_all = sum(class_f1s) / len(class_f1s)
|
|
macro_f1_gold = sum(gold_f1s) / len(gold_f1s)
|
|
assert pytest.approx(macro_f1_all, 1e-6) == metrics["macro_f1_all_22_classes"]
|
|
assert pytest.approx(macro_f1_gold, 1e-6) == metrics["macro_f1_gold_classes"]
|
|
|
|
|
|
def test_golden_vectors_window_merging():
|
|
"""Verify window merging algorithm against all golden test cases."""
|
|
gv_path = pathlib.Path(__file__).parent / "causal_eval_golden_vectors.json"
|
|
with gv_path.open("r", encoding="utf-8") as f:
|
|
golden = json.load(f)
|
|
|
|
merging_data = golden["window_merging_vectors"]
|
|
max_gap = merging_data["max_gap_seconds"]
|
|
|
|
for case in merging_data["cases"]:
|
|
windows = case["windows"]
|
|
merged = merge_temporal_windows(windows, max_gap=max_gap)
|
|
expected = case["expected_merged_segments"]
|
|
|
|
assert len(merged) == len(expected), f"Case {case['case_id']} length mismatch"
|
|
for actual_seg, exp_seg in zip(merged, expected):
|
|
assert pytest.approx(actual_seg["start_abs"], 1e-6) == exp_seg["start_abs"]
|
|
assert pytest.approx(actual_seg["end_abs"], 1e-6) == exp_seg["end_abs"]
|
|
assert actual_seg["label_id"] == exp_seg["label_id"]
|
|
assert actual_seg["label"] == exp_seg["label"]
|