droboticflow/tests/backend/swift_test_utils.py

450 lines
16 KiB
Python

"""ms-swift multimodal JSONL and dataset validation utilities for droboticflow.
Provides:
- Schema validation for ms-swift multimodal conversation JSONL datasets.
- Cross-split zero video leakage detection across train/val/test partitions.
- Strict JSON content validators for Qwen3-VL 200s temporal action localization and 22-class multi-label classification.
- Official 22 convenience store taxonomy lookup and alias resolution.
"""
from __future__ import annotations
import json
import pathlib
import re
from typing import Any
from tests.fixtures.video_fixtures import CONVENIENCE_STORE_22_TAXONOMY
DJI_PATTERN = re.compile(r"(DJI_(?:19|20)\d{6}_\d{4}_D|DJI_(?:19|20)\d{12}_\d{4}_D|DJI_\d+_[A-Z0-9]+_D)", re.IGNORECASE)
ORBBEC_PATTERN = re.compile(r"(Orbbec_Ego_[^/_\\]+?_(?:19|20)\d{6}_\d{6})", re.IGNORECASE)
def resolve_taxonomy_label(query: str) -> tuple[str, str] | None:
"""Resolve an input label, alias, or ID to (canonical_label_id, canonical_label_name).
Returns None if no matching taxonomy label is found.
"""
cleaned = query.strip()
# Check direct ID match ("01" .. "22" or integer 1 .. 22)
id_padded = f"{int(cleaned):02d}" if cleaned.isdigit() and 1 <= int(cleaned) <= 22 else cleaned
if id_padded in CONVENIENCE_STORE_22_TAXONOMY:
return id_padded, CONVENIENCE_STORE_22_TAXONOMY[id_padded]["name"]
# Check exact name match
for lid, meta in CONVENIENCE_STORE_22_TAXONOMY.items():
if meta["name"] == cleaned:
return lid, meta["name"]
# Check alias match
for lid, meta in CONVENIENCE_STORE_22_TAXONOMY.items():
if cleaned in meta.get("aliases", []):
return lid, meta["name"]
for alias in meta.get("aliases", []):
if cleaned in alias or alias in cleaned:
return lid, meta["name"]
return None
def extract_source_video_id(video_path: str) -> str:
"""Extract authoritative source video identifier (e.g. DJI_..._D or Orbbec_Ego_...).
Args:
video_path: Full file path or relative path to a video file.
Returns:
String identifier representing the source root video/capture.
"""
path_str = str(video_path or "").strip()
if not path_str:
return ""
dji_match = DJI_PATTERN.search(path_str)
if dji_match:
return dji_match.group(1)
orbbec_match = ORBBEC_PATTERN.search(path_str)
if orbbec_match:
return orbbec_match.group(1)
# Fallback to parent directory or stem
p = pathlib.PurePosixPath(path_str.replace("\\", "/"))
for part in p.parts:
if part.startswith("DJI_") or part.startswith("Orbbec_Ego_"):
return part
return p.parts[0] if len(p.parts) > 1 else p.stem
def validate_swift_jsonl_schema(file_path: str | pathlib.Path) -> dict[str, Any]:
"""Validate ms-swift multimodal conversation JSONL dataset format.
ms-swift schema:
{
"messages": [
{"role": "user", "content": "<video>...</video>..."},
{"role": "assistant", "content": "..."}
],
"videos": ["/path/to/video.mp4"]
}
Returns:
dict with validation results, errors, total line count, and extracted video paths.
"""
path = pathlib.Path(file_path)
if not path.exists():
return {
"valid": False,
"total_lines": 0,
"valid_lines": 0,
"errors": [f"File not found: {path}"],
"videos": [],
}
errors: list[str] = []
videos: list[str] = []
total_lines = 0
valid_lines = 0
with path.open("r", encoding="utf-8") as handle:
for line_num, line in enumerate(handle, start=1):
stripped = line.strip()
if not stripped:
continue
total_lines += 1
try:
data = json.loads(stripped)
except json.JSONDecodeError as exc:
errors.append(f"Line {line_num}: Invalid JSON syntax - {exc}")
continue
if not isinstance(data, dict):
errors.append(f"Line {line_num}: Top-level record must be a JSON object")
continue
# Validate "messages"
messages = data.get("messages")
if not isinstance(messages, list) or len(messages) < 2:
errors.append(f"Line {line_num}: 'messages' must be a list containing at least 2 messages")
continue
has_user = False
has_assistant = False
for msg_idx, msg in enumerate(messages):
if not isinstance(msg, dict):
errors.append(f"Line {line_num}, message {msg_idx}: Message must be an object")
continue
role = msg.get("role")
content = msg.get("content")
if role not in {"user", "assistant", "system"}:
errors.append(f"Line {line_num}, message {msg_idx}: Invalid role '{role}'")
if not isinstance(content, str) or not content.strip():
errors.append(f"Line {line_num}, message {msg_idx}: 'content' must be non-empty string")
if role == "user":
has_user = True
elif role == "assistant":
has_assistant = True
if not (has_user and has_assistant):
errors.append(f"Line {line_num}: 'messages' must contain both 'user' and 'assistant' roles")
# Validate "videos"
row_videos = data.get("videos")
if row_videos is not None:
if not isinstance(row_videos, list):
errors.append(f"Line {line_num}: 'videos' must be a list of file paths")
else:
for v in row_videos:
if isinstance(v, str) and v.strip():
videos.append(v.strip())
else:
errors.append(f"Line {line_num}: video entry must be a non-empty string path")
if len(errors) == 0 or (len(errors) > 0 and errors[-1].startswith(f"Line {line_num}") is False):
valid_lines += 1
return {
"valid": len(errors) == 0 and total_lines > 0,
"total_lines": total_lines,
"valid_lines": valid_lines,
"errors": errors,
"videos": sorted(set(videos)),
}
def check_zero_video_leakage(
train_path: str | pathlib.Path,
val_path: str | pathlib.Path,
test_path: str | pathlib.Path | None = None,
) -> dict[str, Any]:
"""Verify source video identifiers are strictly partitioned across train/val/test splits without overlap.
Returns:
dict containing leakage status, overlap counts, and overlapping video IDs.
"""
splits_files = {"train": pathlib.Path(train_path), "val": pathlib.Path(val_path)}
if test_path is not None:
splits_files["test"] = pathlib.Path(test_path)
split_sources: dict[str, set[str]] = {}
errors: list[str] = []
for split_name, file_path in splits_files.items():
if not file_path.exists():
errors.append(f"Split file missing: {split_name} ({file_path})")
split_sources[split_name] = set()
continue
sources = set()
with file_path.open("r", encoding="utf-8") as handle:
for line_num, line in enumerate(handle, start=1):
stripped = line.strip()
if not stripped:
continue
try:
data = json.loads(stripped)
# Extract from "videos" list
for v in data.get("videos", []):
src_id = extract_source_video_id(v)
if src_id:
sources.add(src_id)
# Also check user message video tags
for msg in data.get("messages", []):
content = msg.get("content", "")
for tag in re.findall(r"<video>(.*?)</video>", content):
src_id = extract_source_video_id(tag)
if src_id:
sources.add(src_id)
except json.JSONDecodeError:
errors.append(f"{split_name} line {line_num}: invalid JSON")
split_sources[split_name] = sources
train_sources = split_sources.get("train", set())
val_sources = split_sources.get("val", set())
test_sources = split_sources.get("test", set())
train_val_overlap = train_sources & val_sources
train_test_overlap = train_sources & test_sources
val_test_overlap = val_sources & test_sources
all_overlaps = train_val_overlap | train_test_overlap | val_test_overlap
zero_leakage = len(all_overlaps) == 0 and len(errors) == 0
return {
"zero_leakage": zero_leakage,
"leakage_detected": not zero_leakage,
"train_sources_count": len(train_sources),
"val_sources_count": len(val_sources),
"test_sources_count": len(test_sources),
"train_sources": sorted(train_sources),
"val_sources": sorted(val_sources),
"test_sources": sorted(test_sources),
"overlapping_sources": sorted(all_overlaps),
"overlap_details": {
"train_val": sorted(train_val_overlap),
"train_test": sorted(train_test_overlap),
"val_test": sorted(val_test_overlap),
},
"errors": errors,
}
def validate_localization_json_content(
content_str: str,
max_duration: float | None = None,
) -> dict[str, Any]:
"""Validate assistant output JSON string for Qwen3-VL 200s temporal action localization.
Expected schema:
{
"recognized_labels": ["05", "06"],
"segments": [
{"start": 3.5, "end": 44.0, "label_id": "05", "label": "蒸点陈列与出餐"},
{"start": 34.3, "end": 63.3, "label_id": "06", "label": "热食加工与出餐"}
]
}
Or empty:
{"recognized_labels": [], "segments": []}
"""
errors: list[str] = []
if not isinstance(content_str, str) or not content_str.strip():
return {
"valid": False,
"errors": ["Content string is empty or not a string"],
"recognized_labels": [],
"segment_count": 0,
"segments": [],
}
try:
data = json.loads(content_str.strip())
except json.JSONDecodeError as exc:
return {
"valid": False,
"errors": [f"Invalid JSON: {exc}"],
"recognized_labels": [],
"segment_count": 0,
"segments": [],
}
if not isinstance(data, dict):
return {
"valid": False,
"errors": ["Top-level content must be a JSON object"],
"recognized_labels": [],
"segment_count": 0,
"segments": [],
}
recognized_labels = data.get("recognized_labels")
segments = data.get("segments")
if not isinstance(recognized_labels, list):
errors.append("'recognized_labels' must be a list of strings")
recognized_labels = []
if not isinstance(segments, list):
errors.append("'segments' must be a list of segment objects")
segments = []
observed_label_ids: set[str] = set()
validated_segments: list[dict[str, Any]] = []
for idx, seg in enumerate(segments):
if not isinstance(seg, dict):
errors.append(f"Segment {idx} is not an object")
continue
start = seg.get("start")
end = seg.get("end")
label_id = str(seg.get("label_id", "")).strip()
label_name = str(seg.get("label", "")).strip()
# Type and boundary checks
if not isinstance(start, (int, float)) or start < 0:
errors.append(f"Segment {idx}: 'start' must be non-negative number, got {start}")
if not isinstance(end, (int, float)) or (isinstance(start, (int, float)) and end <= start):
errors.append(f"Segment {idx}: 'end' ({end}) must be greater than 'start' ({start})")
if max_duration is not None and isinstance(end, (int, float)) and end > max_duration:
errors.append(f"Segment {idx}: 'end' ({end}s) exceeds window max_duration ({max_duration}s)")
# Taxonomy verification
if label_id not in CONVENIENCE_STORE_22_TAXONOMY:
errors.append(f"Segment {idx}: Unknown label_id '{label_id}' (must be '01'..'22')")
else:
expected_name = CONVENIENCE_STORE_22_TAXONOMY[label_id]["name"]
if label_name and label_name != expected_name:
errors.append(
f"Segment {idx}: Label name mismatch for ID '{label_id}' (expected '{expected_name}', got '{label_name}')"
)
observed_label_ids.add(label_id)
validated_segments.append({
"start": float(start) if isinstance(start, (int, float)) else 0.0,
"end": float(end) if isinstance(end, (int, float)) else 0.0,
"label_id": label_id,
"label": label_name or CONVENIENCE_STORE_22_TAXONOMY.get(label_id, {}).get("name", ""),
})
# Consistency check between recognized_labels and segments
recognized_set = set(str(lid) for lid in recognized_labels)
if recognized_set != observed_label_ids:
errors.append(
f"'recognized_labels' ({sorted(recognized_set)}) does not match unique segment label_ids ({sorted(observed_label_ids)})"
)
return {
"valid": len(errors) == 0,
"errors": errors,
"recognized_labels": sorted(recognized_set),
"segment_count": len(validated_segments),
"segments": validated_segments,
}
def validate_classification_json_content(content_str: str) -> dict[str, Any]:
"""Validate assistant output JSON string for Qwen3-VL 22-class multi-label classification.
Expected schema:
{
"labels": [
{"label_id": "05", "label": "蒸点陈列与出餐"},
{"label_id": "06", "label": "热食加工与出餐"}
]
}
Or empty:
{"labels": []}
"""
errors: list[str] = []
if not isinstance(content_str, str) or not content_str.strip():
return {
"valid": False,
"errors": ["Content string is empty or not a string"],
"label_count": 0,
"labels": [],
}
try:
data = json.loads(content_str.strip())
except json.JSONDecodeError as exc:
return {
"valid": False,
"errors": [f"Invalid JSON: {exc}"],
"label_count": 0,
"labels": [],
}
if not isinstance(data, dict):
return {
"valid": False,
"errors": ["Top-level content must be a JSON object"],
"label_count": 0,
"labels": [],
}
labels_raw = data.get("labels")
if not isinstance(labels_raw, list):
return {
"valid": False,
"errors": ["'labels' must be a list"],
"label_count": 0,
"labels": [],
}
seen_ids: set[str] = set()
validated_labels: list[dict[str, Any]] = []
for idx, item in enumerate(labels_raw):
if not isinstance(item, dict):
errors.append(f"Label {idx} is not an object")
continue
label_id = str(item.get("label_id", "")).strip()
label_name = str(item.get("label", "")).strip()
if label_id in seen_ids:
errors.append(f"Duplicate label_id '{label_id}' in classification output")
seen_ids.add(label_id)
if label_id not in CONVENIENCE_STORE_22_TAXONOMY:
errors.append(f"Label {idx}: Unknown label_id '{label_id}' (must be '01'..'22')")
else:
expected_name = CONVENIENCE_STORE_22_TAXONOMY[label_id]["name"]
if label_name and label_name != expected_name:
errors.append(
f"Label {idx}: Label name mismatch for ID '{label_id}' (expected '{expected_name}', got '{label_name}')"
)
validated_labels.append({
"label_id": label_id,
"label": label_name or CONVENIENCE_STORE_22_TAXONOMY.get(label_id, {}).get("name", ""),
})
return {
"valid": len(errors) == 0,
"errors": errors,
"label_count": len(validated_labels),
"labels": validated_labels,
}