2026-06-24 13:32:11 +08:00

392 lines
14 KiB
Python

from __future__ import annotations
import csv
import json
import pathlib
from typing import Any, Callable
import cv2
import numpy as np
import pyarrow.parquet as pq
HelperMap = dict[str, Callable[..., Any]]
SUPPORTED_TABLE_SUFFIXES = {".csv", ".json", ".jsonl", ".parquet"}
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
VIDEO_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
DEPTH_SUFFIXES = {".npy", ".tif", ".tiff"}
POINTCLOUD_SUFFIXES = {".pcd", ".ply", ".las", ".laz"}
MEDIA_SUFFIXES = IMAGE_SUFFIXES | VIDEO_SUFFIXES | DEPTH_SUFFIXES | POINTCLOUD_SUFFIXES
DEFAULT_MEDIA_FIELDS = (
"image",
"image_path",
"imagePath",
"video",
"video_path",
"videoPath",
"rgb",
"rgb_path",
"depth",
"depth_path",
"mask",
"mask_path",
"pointcloud",
"pointcloud_path",
"observation.image",
"observation.rgb",
"observation.depth",
"observation.video",
"observation.images.rgb",
"observation.images.wrist",
)
def _int_param(params: dict[str, str], key: str, default: int) -> int:
raw = str(params.get(key, str(default)) or str(default)).strip()
try:
value = int(raw)
except ValueError as exc:
raise RuntimeError(f"{key} must be an integer") from exc
if value <= 0:
raise RuntimeError(f"{key} must be > 0")
return value
def _bool_param(params: dict[str, str], key: str, default: bool) -> bool:
raw = str(params.get(key, "") or "").strip().lower()
if not raw:
return default
if raw in {"1", "true", "yes", "on"}:
return True
if raw in {"0", "false", "no", "off"}:
return False
raise RuntimeError(f"{key} must be a boolean")
def _field_list(raw: str, default: tuple[str, ...]) -> list[str]:
values = [item.strip() for item in str(raw or "").split(",") if item.strip()]
return values or list(default)
def _load_json(path: pathlib.Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise RuntimeError(f"{path} is not valid JSON") from exc
def _load_jsonl(path: pathlib.Path, max_rows: int) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8") as handle:
for line_number, line in enumerate(handle, start=1):
if len(rows) >= max_rows:
break
stripped = line.strip()
if not stripped:
continue
try:
payload = json.loads(stripped)
except json.JSONDecodeError as exc:
raise RuntimeError(f"{path} line {line_number} is not valid JSON") from exc
rows.extend(_rows_from_json(payload))
return rows[:max_rows]
def _rows_from_json(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, dict):
for key in ("items", "rows", "frames", "steps", "episodes", "trajectories", "demos"):
value = payload.get(key)
if isinstance(value, list):
if key in {"episodes", "trajectories", "demos"}:
rows: list[dict[str, Any]] = []
for episode_index, episode in enumerate(value):
if not isinstance(episode, dict):
continue
steps = episode.get("steps") or episode.get("frames") or episode.get("trajectory")
if isinstance(steps, list):
episode_id = episode.get("episode_id", episode.get("episodeId", episode_index))
for step_index, step in enumerate(steps):
if isinstance(step, dict):
row = dict(step)
row.setdefault("episode_id", episode_id)
row.setdefault("frame_index", step_index)
rows.append(row)
return rows
return [item for item in value if isinstance(item, dict)]
return [payload]
if isinstance(payload, list):
return [item for item in payload if isinstance(item, dict)]
return []
def _candidate_files(source_path: pathlib.Path, max_files: int) -> list[pathlib.Path]:
if source_path.is_file():
return [source_path] if source_path.suffix.lower() in SUPPORTED_TABLE_SUFFIXES else []
preferred = [
source_path / "manifest.parquet",
source_path / "manifest.jsonl",
source_path / "manifest.csv",
source_path / "episodes.jsonl",
source_path / "meta" / "episodes.jsonl",
]
files: list[pathlib.Path] = []
seen: set[pathlib.Path] = set()
for path in preferred:
if path.is_file() and path.suffix.lower() in SUPPORTED_TABLE_SUFFIXES:
files.append(path)
seen.add(path.resolve())
for candidate in sorted(source_path.rglob("*")):
if len(files) >= max_files:
break
if not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_TABLE_SUFFIXES:
continue
resolved = candidate.resolve()
if resolved in seen:
continue
files.append(candidate)
seen.add(resolved)
return files[:max_files]
def _rows_from_file(path: pathlib.Path, max_rows: int) -> list[dict[str, Any]]:
suffix = path.suffix.lower()
if suffix == ".csv":
with path.open("r", encoding="utf-8", newline="") as handle:
return [dict(row) for _, row in zip(range(max_rows), csv.DictReader(handle))]
if suffix == ".json":
return _rows_from_json(_load_json(path))[:max_rows]
if suffix == ".jsonl":
return _load_jsonl(path, max_rows)
if suffix == ".parquet":
rows: list[dict[str, Any]] = []
parquet_file = pq.ParquetFile(path)
for batch in parquet_file.iter_batches(batch_size=min(max(max_rows, 1), 4096)):
rows.extend(row for row in batch.to_pylist() if isinstance(row, dict))
if len(rows) >= max_rows:
break
return rows[:max_rows]
return []
def _get_path_value(payload: dict[str, Any], path: str) -> Any:
if path in payload:
return payload[path]
current: Any = payload
for part in path.split("."):
if isinstance(current, dict) and part in current:
current = current[part]
else:
return None
return current
def _flatten_strings(value: Any) -> list[str]:
if isinstance(value, str):
return [value.strip()] if value.strip() else []
if isinstance(value, dict):
result: list[str] = []
for item in value.values():
result.extend(_flatten_strings(item))
return result
if isinstance(value, (list, tuple)):
result: list[str] = []
for item in value:
result.extend(_flatten_strings(item))
return result
return []
def _looks_like_media_field(key: str) -> bool:
key_lower = key.lower()
return any(token in key_lower for token in ("image", "video", "rgb", "depth", "mask", "point", "file", "path"))
def _row_references(row: dict[str, Any], fields: list[str], infer_fields: bool) -> list[dict[str, str]]:
refs: list[dict[str, str]] = []
for field in fields:
for value in _flatten_strings(_get_path_value(row, field)):
if pathlib.PurePosixPath(value).suffix.lower() in MEDIA_SUFFIXES:
refs.append({"field": field, "path": value})
if infer_fields:
for key, value in row.items():
if not _looks_like_media_field(str(key)):
continue
for candidate in _flatten_strings(value):
if pathlib.PurePosixPath(candidate).suffix.lower() in MEDIA_SUFFIXES:
refs.append({"field": str(key), "path": candidate})
seen: set[tuple[str, str]] = set()
unique: list[dict[str, str]] = []
for ref in refs:
key = (ref["field"], ref["path"])
if key in seen:
continue
unique.append(ref)
seen.add(key)
return unique
def _resolve_path(root: pathlib.Path, value: str) -> pathlib.Path:
raw = str(value).strip()
path = pathlib.Path(raw)
if path.is_absolute():
return path
for base in (root, root / "data", root / "images", root / "videos", root / "media"):
candidate = base / path
if candidate.exists():
return candidate
return root / path
def _inspect_file(path: pathlib.Path, helpers: HelperMap, deep_check: bool) -> dict[str, Any]:
suffix = path.suffix.lower()
payload: dict[str, Any] = {"path": str(path), "exists": path.exists(), "suffix": suffix, "readable": False}
if not path.exists() or not path.is_file():
return payload
payload["sizeBytes"] = path.stat().st_size
if not deep_check:
payload["readable"] = True
return payload
if suffix in IMAGE_SUFFIXES:
image = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
payload["readable"] = image is not None
if image is not None:
payload["shape"] = [int(item) for item in image.shape]
return payload
if suffix in DEPTH_SUFFIXES:
if suffix == ".npy":
data = np.load(path)
payload["readable"] = True
payload["shape"] = [int(item) for item in data.shape]
payload["dtype"] = str(data.dtype)
else:
image = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
payload["readable"] = image is not None
if image is not None:
payload["shape"] = [int(item) for item in image.shape]
return payload
if suffix in VIDEO_SUFFIXES:
try:
metrics = helpers["inspect_video_file"](path)
payload["readable"] = int(metrics.get("frameCount") or 0) > 0
payload["video"] = metrics
except Exception as exc:
payload["error"] = str(exc)
return payload
if suffix in POINTCLOUD_SUFFIXES:
with path.open("rb") as handle:
head = handle.read(128)
payload["readable"] = bool(head)
payload["headerPreview"] = head[:64].decode("utf-8", errors="replace")
return payload
payload["readable"] = True
return payload
def run(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
source_path = helpers["resolve_source_path"](params, inputs)
if source_path is None:
raise RuntimeError("媒体引用可读性校验需要 SOURCE_PATH 指向真实 manifest 或数据集目录")
max_files = _int_param(params, "MAX_FILES", 50)
max_rows = _int_param(params, "MAX_ROWS", 200000)
max_references = _int_param(params, "MAX_REFERENCES", 5000)
deep_check = _bool_param(params, "DEEP_CHECK", True)
infer_fields = _bool_param(params, "INFER_FIELDS", True)
fields = _field_list(params.get("MEDIA_FIELDS", ""), DEFAULT_MEDIA_FIELDS)
root = source_path if source_path.is_dir() else source_path.parent
files = _candidate_files(source_path, max_files)
if not files:
raise RuntimeError(f"{source_path} does not contain readable media manifest CSV/JSON/JSONL/Parquet files")
rows: list[dict[str, Any]] = []
parsed_files: list[str] = []
for file_path in files:
if len(rows) >= max_rows:
break
next_rows = _rows_from_file(file_path, max(1, max_rows - len(rows)))
if next_rows:
rows.extend(next_rows)
parsed_files.append(str(file_path))
if not rows:
raise RuntimeError(f"{source_path} media manifest files are empty")
references: list[dict[str, Any]] = []
field_counts: dict[str, int] = {}
missing: list[dict[str, Any]] = []
unreadable: list[dict[str, Any]] = []
inspected_cache: dict[str, dict[str, Any]] = {}
for row_index, row in enumerate(rows):
for ref in _row_references(row, fields, infer_fields):
if len(references) >= max_references:
break
resolved = _resolve_path(root, ref["path"])
cache_key = str(resolved)
inspected = inspected_cache.get(cache_key)
if inspected is None:
inspected = _inspect_file(resolved, helpers, deep_check)
inspected_cache[cache_key] = inspected
item = {
"rowIndex": row_index,
"field": ref["field"],
"reference": ref["path"],
"resolvedPath": str(resolved),
"exists": inspected.get("exists", False),
"readable": inspected.get("readable", False),
"suffix": inspected.get("suffix", ""),
"sizeBytes": inspected.get("sizeBytes", 0),
}
references.append(item)
field_counts[ref["field"]] = field_counts.get(ref["field"], 0) + 1
if not item["exists"]:
missing.append(item)
elif not item["readable"]:
unreadable.append(item)
if len(references) >= max_references:
break
issues: list[dict[str, Any]] = []
warnings: list[dict[str, Any]] = []
if not references:
issues.append({"code": "no_media_references_found", "fields": fields})
if missing:
issues.append({"code": "missing_media_files", "count": len(missing), "examples": missing[:100]})
if unreadable:
issues.append({"code": "unreadable_media_files", "count": len(unreadable), "examples": unreadable[:100]})
if len(references) >= max_references:
warnings.append({"code": "reference_scan_truncated", "maxReferences": max_references})
payload = {
"datasetType": "media-reference-validation",
"sourcePath": str(source_path),
"valid": not issues,
"issueCount": len(issues),
"warningCount": len(warnings),
"issues": issues,
"warnings": warnings,
"parsedFiles": parsed_files,
"rowCount": len(rows),
"referenceCount": len(references),
"uniqueResolvedPathCount": len(inspected_cache),
"missingCount": len(missing),
"unreadableCount": len(unreadable),
"fieldCounts": field_counts,
"references": references[:500],
}
return [
helpers["write_json_artifact"](
run_id,
node_id,
output_dir,
"media-reference-validation.json",
payload,
output=helpers["output_binding"](node_data),
)
]