450 lines
16 KiB
Python
450 lines
16 KiB
Python
"""ms-swift multimodal JSONL and dataset validation utilities for droboticflow.
|
|
|
|
Provides:
|
|
- Schema validation for ms-swift multimodal conversation JSONL datasets.
|
|
- Cross-split zero video leakage detection across train/val/test partitions.
|
|
- Strict JSON content validators for Qwen3-VL 200s temporal action localization and 22-class multi-label classification.
|
|
- Official 22 convenience store taxonomy lookup and alias resolution.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import pathlib
|
|
import re
|
|
from typing import Any
|
|
|
|
from tests.fixtures.video_fixtures import CONVENIENCE_STORE_22_TAXONOMY
|
|
|
|
DJI_PATTERN = re.compile(r"(DJI_(?:19|20)\d{6}_\d{4}_D|DJI_(?:19|20)\d{12}_\d{4}_D|DJI_\d+_[A-Z0-9]+_D)", re.IGNORECASE)
|
|
ORBBEC_PATTERN = re.compile(r"(Orbbec_Ego_[^/_\\]+?_(?:19|20)\d{6}_\d{6})", re.IGNORECASE)
|
|
|
|
|
|
def resolve_taxonomy_label(query: str) -> tuple[str, str] | None:
|
|
"""Resolve an input label, alias, or ID to (canonical_label_id, canonical_label_name).
|
|
|
|
Returns None if no matching taxonomy label is found.
|
|
"""
|
|
cleaned = query.strip()
|
|
# Check direct ID match ("01" .. "22" or integer 1 .. 22)
|
|
id_padded = f"{int(cleaned):02d}" if cleaned.isdigit() and 1 <= int(cleaned) <= 22 else cleaned
|
|
if id_padded in CONVENIENCE_STORE_22_TAXONOMY:
|
|
return id_padded, CONVENIENCE_STORE_22_TAXONOMY[id_padded]["name"]
|
|
|
|
# Check exact name match
|
|
for lid, meta in CONVENIENCE_STORE_22_TAXONOMY.items():
|
|
if meta["name"] == cleaned:
|
|
return lid, meta["name"]
|
|
|
|
# Check alias match
|
|
for lid, meta in CONVENIENCE_STORE_22_TAXONOMY.items():
|
|
if cleaned in meta.get("aliases", []):
|
|
return lid, meta["name"]
|
|
for alias in meta.get("aliases", []):
|
|
if cleaned in alias or alias in cleaned:
|
|
return lid, meta["name"]
|
|
|
|
return None
|
|
|
|
|
|
def extract_source_video_id(video_path: str) -> str:
|
|
"""Extract authoritative source video identifier (e.g. DJI_..._D or Orbbec_Ego_...).
|
|
|
|
Args:
|
|
video_path: Full file path or relative path to a video file.
|
|
|
|
Returns:
|
|
String identifier representing the source root video/capture.
|
|
"""
|
|
path_str = str(video_path or "").strip()
|
|
if not path_str:
|
|
return ""
|
|
|
|
dji_match = DJI_PATTERN.search(path_str)
|
|
if dji_match:
|
|
return dji_match.group(1)
|
|
|
|
orbbec_match = ORBBEC_PATTERN.search(path_str)
|
|
if orbbec_match:
|
|
return orbbec_match.group(1)
|
|
|
|
# Fallback to parent directory or stem
|
|
p = pathlib.PurePosixPath(path_str.replace("\\", "/"))
|
|
for part in p.parts:
|
|
if part.startswith("DJI_") or part.startswith("Orbbec_Ego_"):
|
|
return part
|
|
|
|
return p.parts[0] if len(p.parts) > 1 else p.stem
|
|
|
|
|
|
def validate_swift_jsonl_schema(file_path: str | pathlib.Path) -> dict[str, Any]:
|
|
"""Validate ms-swift multimodal conversation JSONL dataset format.
|
|
|
|
ms-swift schema:
|
|
{
|
|
"messages": [
|
|
{"role": "user", "content": "<video>...</video>..."},
|
|
{"role": "assistant", "content": "..."}
|
|
],
|
|
"videos": ["/path/to/video.mp4"]
|
|
}
|
|
|
|
Returns:
|
|
dict with validation results, errors, total line count, and extracted video paths.
|
|
"""
|
|
path = pathlib.Path(file_path)
|
|
if not path.exists():
|
|
return {
|
|
"valid": False,
|
|
"total_lines": 0,
|
|
"valid_lines": 0,
|
|
"errors": [f"File not found: {path}"],
|
|
"videos": [],
|
|
}
|
|
|
|
errors: list[str] = []
|
|
videos: list[str] = []
|
|
total_lines = 0
|
|
valid_lines = 0
|
|
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
for line_num, line in enumerate(handle, start=1):
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
continue
|
|
total_lines += 1
|
|
try:
|
|
data = json.loads(stripped)
|
|
except json.JSONDecodeError as exc:
|
|
errors.append(f"Line {line_num}: Invalid JSON syntax - {exc}")
|
|
continue
|
|
|
|
if not isinstance(data, dict):
|
|
errors.append(f"Line {line_num}: Top-level record must be a JSON object")
|
|
continue
|
|
|
|
# Validate "messages"
|
|
messages = data.get("messages")
|
|
if not isinstance(messages, list) or len(messages) < 2:
|
|
errors.append(f"Line {line_num}: 'messages' must be a list containing at least 2 messages")
|
|
continue
|
|
|
|
has_user = False
|
|
has_assistant = False
|
|
for msg_idx, msg in enumerate(messages):
|
|
if not isinstance(msg, dict):
|
|
errors.append(f"Line {line_num}, message {msg_idx}: Message must be an object")
|
|
continue
|
|
role = msg.get("role")
|
|
content = msg.get("content")
|
|
if role not in {"user", "assistant", "system"}:
|
|
errors.append(f"Line {line_num}, message {msg_idx}: Invalid role '{role}'")
|
|
if not isinstance(content, str) or not content.strip():
|
|
errors.append(f"Line {line_num}, message {msg_idx}: 'content' must be non-empty string")
|
|
if role == "user":
|
|
has_user = True
|
|
elif role == "assistant":
|
|
has_assistant = True
|
|
|
|
if not (has_user and has_assistant):
|
|
errors.append(f"Line {line_num}: 'messages' must contain both 'user' and 'assistant' roles")
|
|
|
|
# Validate "videos"
|
|
row_videos = data.get("videos")
|
|
if row_videos is not None:
|
|
if not isinstance(row_videos, list):
|
|
errors.append(f"Line {line_num}: 'videos' must be a list of file paths")
|
|
else:
|
|
for v in row_videos:
|
|
if isinstance(v, str) and v.strip():
|
|
videos.append(v.strip())
|
|
else:
|
|
errors.append(f"Line {line_num}: video entry must be a non-empty string path")
|
|
|
|
if len(errors) == 0 or (len(errors) > 0 and errors[-1].startswith(f"Line {line_num}") is False):
|
|
valid_lines += 1
|
|
|
|
return {
|
|
"valid": len(errors) == 0 and total_lines > 0,
|
|
"total_lines": total_lines,
|
|
"valid_lines": valid_lines,
|
|
"errors": errors,
|
|
"videos": sorted(set(videos)),
|
|
}
|
|
|
|
|
|
def check_zero_video_leakage(
|
|
train_path: str | pathlib.Path,
|
|
val_path: str | pathlib.Path,
|
|
test_path: str | pathlib.Path | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Verify source video identifiers are strictly partitioned across train/val/test splits without overlap.
|
|
|
|
Returns:
|
|
dict containing leakage status, overlap counts, and overlapping video IDs.
|
|
"""
|
|
splits_files = {"train": pathlib.Path(train_path), "val": pathlib.Path(val_path)}
|
|
if test_path is not None:
|
|
splits_files["test"] = pathlib.Path(test_path)
|
|
|
|
split_sources: dict[str, set[str]] = {}
|
|
errors: list[str] = []
|
|
|
|
for split_name, file_path in splits_files.items():
|
|
if not file_path.exists():
|
|
errors.append(f"Split file missing: {split_name} ({file_path})")
|
|
split_sources[split_name] = set()
|
|
continue
|
|
|
|
sources = set()
|
|
with file_path.open("r", encoding="utf-8") as handle:
|
|
for line_num, line in enumerate(handle, start=1):
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
continue
|
|
try:
|
|
data = json.loads(stripped)
|
|
# Extract from "videos" list
|
|
for v in data.get("videos", []):
|
|
src_id = extract_source_video_id(v)
|
|
if src_id:
|
|
sources.add(src_id)
|
|
# Also check user message video tags
|
|
for msg in data.get("messages", []):
|
|
content = msg.get("content", "")
|
|
for tag in re.findall(r"<video>(.*?)</video>", content):
|
|
src_id = extract_source_video_id(tag)
|
|
if src_id:
|
|
sources.add(src_id)
|
|
except json.JSONDecodeError:
|
|
errors.append(f"{split_name} line {line_num}: invalid JSON")
|
|
|
|
split_sources[split_name] = sources
|
|
|
|
train_sources = split_sources.get("train", set())
|
|
val_sources = split_sources.get("val", set())
|
|
test_sources = split_sources.get("test", set())
|
|
|
|
train_val_overlap = train_sources & val_sources
|
|
train_test_overlap = train_sources & test_sources
|
|
val_test_overlap = val_sources & test_sources
|
|
all_overlaps = train_val_overlap | train_test_overlap | val_test_overlap
|
|
|
|
zero_leakage = len(all_overlaps) == 0 and len(errors) == 0
|
|
|
|
return {
|
|
"zero_leakage": zero_leakage,
|
|
"leakage_detected": not zero_leakage,
|
|
"train_sources_count": len(train_sources),
|
|
"val_sources_count": len(val_sources),
|
|
"test_sources_count": len(test_sources),
|
|
"train_sources": sorted(train_sources),
|
|
"val_sources": sorted(val_sources),
|
|
"test_sources": sorted(test_sources),
|
|
"overlapping_sources": sorted(all_overlaps),
|
|
"overlap_details": {
|
|
"train_val": sorted(train_val_overlap),
|
|
"train_test": sorted(train_test_overlap),
|
|
"val_test": sorted(val_test_overlap),
|
|
},
|
|
"errors": errors,
|
|
}
|
|
|
|
|
|
def validate_localization_json_content(
|
|
content_str: str,
|
|
max_duration: float | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Validate assistant output JSON string for Qwen3-VL 200s temporal action localization.
|
|
|
|
Expected schema:
|
|
{
|
|
"recognized_labels": ["05", "06"],
|
|
"segments": [
|
|
{"start": 3.5, "end": 44.0, "label_id": "05", "label": "蒸点陈列与出餐"},
|
|
{"start": 34.3, "end": 63.3, "label_id": "06", "label": "热食加工与出餐"}
|
|
]
|
|
}
|
|
Or empty:
|
|
{"recognized_labels": [], "segments": []}
|
|
"""
|
|
errors: list[str] = []
|
|
if not isinstance(content_str, str) or not content_str.strip():
|
|
return {
|
|
"valid": False,
|
|
"errors": ["Content string is empty or not a string"],
|
|
"recognized_labels": [],
|
|
"segment_count": 0,
|
|
"segments": [],
|
|
}
|
|
|
|
try:
|
|
data = json.loads(content_str.strip())
|
|
except json.JSONDecodeError as exc:
|
|
return {
|
|
"valid": False,
|
|
"errors": [f"Invalid JSON: {exc}"],
|
|
"recognized_labels": [],
|
|
"segment_count": 0,
|
|
"segments": [],
|
|
}
|
|
|
|
if not isinstance(data, dict):
|
|
return {
|
|
"valid": False,
|
|
"errors": ["Top-level content must be a JSON object"],
|
|
"recognized_labels": [],
|
|
"segment_count": 0,
|
|
"segments": [],
|
|
}
|
|
|
|
recognized_labels = data.get("recognized_labels")
|
|
segments = data.get("segments")
|
|
|
|
if not isinstance(recognized_labels, list):
|
|
errors.append("'recognized_labels' must be a list of strings")
|
|
recognized_labels = []
|
|
|
|
if not isinstance(segments, list):
|
|
errors.append("'segments' must be a list of segment objects")
|
|
segments = []
|
|
|
|
observed_label_ids: set[str] = set()
|
|
validated_segments: list[dict[str, Any]] = []
|
|
|
|
for idx, seg in enumerate(segments):
|
|
if not isinstance(seg, dict):
|
|
errors.append(f"Segment {idx} is not an object")
|
|
continue
|
|
start = seg.get("start")
|
|
end = seg.get("end")
|
|
label_id = str(seg.get("label_id", "")).strip()
|
|
label_name = str(seg.get("label", "")).strip()
|
|
|
|
# Type and boundary checks
|
|
if not isinstance(start, (int, float)) or start < 0:
|
|
errors.append(f"Segment {idx}: 'start' must be non-negative number, got {start}")
|
|
if not isinstance(end, (int, float)) or (isinstance(start, (int, float)) and end <= start):
|
|
errors.append(f"Segment {idx}: 'end' ({end}) must be greater than 'start' ({start})")
|
|
|
|
if max_duration is not None and isinstance(end, (int, float)) and end > max_duration:
|
|
errors.append(f"Segment {idx}: 'end' ({end}s) exceeds window max_duration ({max_duration}s)")
|
|
|
|
# Taxonomy verification
|
|
if label_id not in CONVENIENCE_STORE_22_TAXONOMY:
|
|
errors.append(f"Segment {idx}: Unknown label_id '{label_id}' (must be '01'..'22')")
|
|
else:
|
|
expected_name = CONVENIENCE_STORE_22_TAXONOMY[label_id]["name"]
|
|
if label_name and label_name != expected_name:
|
|
errors.append(
|
|
f"Segment {idx}: Label name mismatch for ID '{label_id}' (expected '{expected_name}', got '{label_name}')"
|
|
)
|
|
|
|
observed_label_ids.add(label_id)
|
|
validated_segments.append({
|
|
"start": float(start) if isinstance(start, (int, float)) else 0.0,
|
|
"end": float(end) if isinstance(end, (int, float)) else 0.0,
|
|
"label_id": label_id,
|
|
"label": label_name or CONVENIENCE_STORE_22_TAXONOMY.get(label_id, {}).get("name", ""),
|
|
})
|
|
|
|
# Consistency check between recognized_labels and segments
|
|
recognized_set = set(str(lid) for lid in recognized_labels)
|
|
if recognized_set != observed_label_ids:
|
|
errors.append(
|
|
f"'recognized_labels' ({sorted(recognized_set)}) does not match unique segment label_ids ({sorted(observed_label_ids)})"
|
|
)
|
|
|
|
return {
|
|
"valid": len(errors) == 0,
|
|
"errors": errors,
|
|
"recognized_labels": sorted(recognized_set),
|
|
"segment_count": len(validated_segments),
|
|
"segments": validated_segments,
|
|
}
|
|
|
|
|
|
def validate_classification_json_content(content_str: str) -> dict[str, Any]:
|
|
"""Validate assistant output JSON string for Qwen3-VL 22-class multi-label classification.
|
|
|
|
Expected schema:
|
|
{
|
|
"labels": [
|
|
{"label_id": "05", "label": "蒸点陈列与出餐"},
|
|
{"label_id": "06", "label": "热食加工与出餐"}
|
|
]
|
|
}
|
|
Or empty:
|
|
{"labels": []}
|
|
"""
|
|
errors: list[str] = []
|
|
if not isinstance(content_str, str) or not content_str.strip():
|
|
return {
|
|
"valid": False,
|
|
"errors": ["Content string is empty or not a string"],
|
|
"label_count": 0,
|
|
"labels": [],
|
|
}
|
|
|
|
try:
|
|
data = json.loads(content_str.strip())
|
|
except json.JSONDecodeError as exc:
|
|
return {
|
|
"valid": False,
|
|
"errors": [f"Invalid JSON: {exc}"],
|
|
"label_count": 0,
|
|
"labels": [],
|
|
}
|
|
|
|
if not isinstance(data, dict):
|
|
return {
|
|
"valid": False,
|
|
"errors": ["Top-level content must be a JSON object"],
|
|
"label_count": 0,
|
|
"labels": [],
|
|
}
|
|
|
|
labels_raw = data.get("labels")
|
|
if not isinstance(labels_raw, list):
|
|
return {
|
|
"valid": False,
|
|
"errors": ["'labels' must be a list"],
|
|
"label_count": 0,
|
|
"labels": [],
|
|
}
|
|
|
|
seen_ids: set[str] = set()
|
|
validated_labels: list[dict[str, Any]] = []
|
|
|
|
for idx, item in enumerate(labels_raw):
|
|
if not isinstance(item, dict):
|
|
errors.append(f"Label {idx} is not an object")
|
|
continue
|
|
label_id = str(item.get("label_id", "")).strip()
|
|
label_name = str(item.get("label", "")).strip()
|
|
|
|
if label_id in seen_ids:
|
|
errors.append(f"Duplicate label_id '{label_id}' in classification output")
|
|
seen_ids.add(label_id)
|
|
|
|
if label_id not in CONVENIENCE_STORE_22_TAXONOMY:
|
|
errors.append(f"Label {idx}: Unknown label_id '{label_id}' (must be '01'..'22')")
|
|
else:
|
|
expected_name = CONVENIENCE_STORE_22_TAXONOMY[label_id]["name"]
|
|
if label_name and label_name != expected_name:
|
|
errors.append(
|
|
f"Label {idx}: Label name mismatch for ID '{label_id}' (expected '{expected_name}', got '{label_name}')"
|
|
)
|
|
|
|
validated_labels.append({
|
|
"label_id": label_id,
|
|
"label": label_name or CONVENIENCE_STORE_22_TAXONOMY.get(label_id, {}).get("name", ""),
|
|
})
|
|
|
|
return {
|
|
"valid": len(errors) == 0,
|
|
"errors": errors,
|
|
"label_count": len(validated_labels),
|
|
"labels": validated_labels,
|
|
}
|