forked from longtao.wu/droboticflow
392 lines
14 KiB
Python
392 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import pathlib
|
|
from typing import Any, Callable
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import pyarrow.parquet as pq
|
|
|
|
|
|
HelperMap = dict[str, Callable[..., Any]]
|
|
SUPPORTED_TABLE_SUFFIXES = {".csv", ".json", ".jsonl", ".parquet"}
|
|
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
|
|
VIDEO_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
|
|
DEPTH_SUFFIXES = {".npy", ".tif", ".tiff"}
|
|
POINTCLOUD_SUFFIXES = {".pcd", ".ply", ".las", ".laz"}
|
|
MEDIA_SUFFIXES = IMAGE_SUFFIXES | VIDEO_SUFFIXES | DEPTH_SUFFIXES | POINTCLOUD_SUFFIXES
|
|
DEFAULT_MEDIA_FIELDS = (
|
|
"image",
|
|
"image_path",
|
|
"imagePath",
|
|
"video",
|
|
"video_path",
|
|
"videoPath",
|
|
"rgb",
|
|
"rgb_path",
|
|
"depth",
|
|
"depth_path",
|
|
"mask",
|
|
"mask_path",
|
|
"pointcloud",
|
|
"pointcloud_path",
|
|
"observation.image",
|
|
"observation.rgb",
|
|
"observation.depth",
|
|
"observation.video",
|
|
"observation.images.rgb",
|
|
"observation.images.wrist",
|
|
)
|
|
|
|
|
|
def _int_param(params: dict[str, str], key: str, default: int) -> int:
|
|
raw = str(params.get(key, str(default)) or str(default)).strip()
|
|
try:
|
|
value = int(raw)
|
|
except ValueError as exc:
|
|
raise RuntimeError(f"{key} must be an integer") from exc
|
|
if value <= 0:
|
|
raise RuntimeError(f"{key} must be > 0")
|
|
return value
|
|
|
|
|
|
def _bool_param(params: dict[str, str], key: str, default: bool) -> bool:
|
|
raw = str(params.get(key, "") or "").strip().lower()
|
|
if not raw:
|
|
return default
|
|
if raw in {"1", "true", "yes", "on"}:
|
|
return True
|
|
if raw in {"0", "false", "no", "off"}:
|
|
return False
|
|
raise RuntimeError(f"{key} must be a boolean")
|
|
|
|
|
|
def _field_list(raw: str, default: tuple[str, ...]) -> list[str]:
|
|
values = [item.strip() for item in str(raw or "").split(",") if item.strip()]
|
|
return values or list(default)
|
|
|
|
|
|
def _load_json(path: pathlib.Path) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(f"{path} is not valid JSON") from exc
|
|
|
|
|
|
def _load_jsonl(path: pathlib.Path, max_rows: int) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
if len(rows) >= max_rows:
|
|
break
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
continue
|
|
try:
|
|
payload = json.loads(stripped)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(f"{path} line {line_number} is not valid JSON") from exc
|
|
rows.extend(_rows_from_json(payload))
|
|
return rows[:max_rows]
|
|
|
|
|
|
def _rows_from_json(payload: Any) -> list[dict[str, Any]]:
|
|
if isinstance(payload, dict):
|
|
for key in ("items", "rows", "frames", "steps", "episodes", "trajectories", "demos"):
|
|
value = payload.get(key)
|
|
if isinstance(value, list):
|
|
if key in {"episodes", "trajectories", "demos"}:
|
|
rows: list[dict[str, Any]] = []
|
|
for episode_index, episode in enumerate(value):
|
|
if not isinstance(episode, dict):
|
|
continue
|
|
steps = episode.get("steps") or episode.get("frames") or episode.get("trajectory")
|
|
if isinstance(steps, list):
|
|
episode_id = episode.get("episode_id", episode.get("episodeId", episode_index))
|
|
for step_index, step in enumerate(steps):
|
|
if isinstance(step, dict):
|
|
row = dict(step)
|
|
row.setdefault("episode_id", episode_id)
|
|
row.setdefault("frame_index", step_index)
|
|
rows.append(row)
|
|
return rows
|
|
return [item for item in value if isinstance(item, dict)]
|
|
return [payload]
|
|
if isinstance(payload, list):
|
|
return [item for item in payload if isinstance(item, dict)]
|
|
return []
|
|
|
|
|
|
def _candidate_files(source_path: pathlib.Path, max_files: int) -> list[pathlib.Path]:
|
|
if source_path.is_file():
|
|
return [source_path] if source_path.suffix.lower() in SUPPORTED_TABLE_SUFFIXES else []
|
|
preferred = [
|
|
source_path / "manifest.parquet",
|
|
source_path / "manifest.jsonl",
|
|
source_path / "manifest.csv",
|
|
source_path / "episodes.jsonl",
|
|
source_path / "meta" / "episodes.jsonl",
|
|
]
|
|
files: list[pathlib.Path] = []
|
|
seen: set[pathlib.Path] = set()
|
|
for path in preferred:
|
|
if path.is_file() and path.suffix.lower() in SUPPORTED_TABLE_SUFFIXES:
|
|
files.append(path)
|
|
seen.add(path.resolve())
|
|
for candidate in sorted(source_path.rglob("*")):
|
|
if len(files) >= max_files:
|
|
break
|
|
if not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_TABLE_SUFFIXES:
|
|
continue
|
|
resolved = candidate.resolve()
|
|
if resolved in seen:
|
|
continue
|
|
files.append(candidate)
|
|
seen.add(resolved)
|
|
return files[:max_files]
|
|
|
|
|
|
def _rows_from_file(path: pathlib.Path, max_rows: int) -> list[dict[str, Any]]:
|
|
suffix = path.suffix.lower()
|
|
if suffix == ".csv":
|
|
with path.open("r", encoding="utf-8", newline="") as handle:
|
|
return [dict(row) for _, row in zip(range(max_rows), csv.DictReader(handle))]
|
|
if suffix == ".json":
|
|
return _rows_from_json(_load_json(path))[:max_rows]
|
|
if suffix == ".jsonl":
|
|
return _load_jsonl(path, max_rows)
|
|
if suffix == ".parquet":
|
|
rows: list[dict[str, Any]] = []
|
|
parquet_file = pq.ParquetFile(path)
|
|
for batch in parquet_file.iter_batches(batch_size=min(max(max_rows, 1), 4096)):
|
|
rows.extend(row for row in batch.to_pylist() if isinstance(row, dict))
|
|
if len(rows) >= max_rows:
|
|
break
|
|
return rows[:max_rows]
|
|
return []
|
|
|
|
|
|
def _get_path_value(payload: dict[str, Any], path: str) -> Any:
|
|
if path in payload:
|
|
return payload[path]
|
|
current: Any = payload
|
|
for part in path.split("."):
|
|
if isinstance(current, dict) and part in current:
|
|
current = current[part]
|
|
else:
|
|
return None
|
|
return current
|
|
|
|
|
|
def _flatten_strings(value: Any) -> list[str]:
|
|
if isinstance(value, str):
|
|
return [value.strip()] if value.strip() else []
|
|
if isinstance(value, dict):
|
|
result: list[str] = []
|
|
for item in value.values():
|
|
result.extend(_flatten_strings(item))
|
|
return result
|
|
if isinstance(value, (list, tuple)):
|
|
result: list[str] = []
|
|
for item in value:
|
|
result.extend(_flatten_strings(item))
|
|
return result
|
|
return []
|
|
|
|
|
|
def _looks_like_media_field(key: str) -> bool:
|
|
key_lower = key.lower()
|
|
return any(token in key_lower for token in ("image", "video", "rgb", "depth", "mask", "point", "file", "path"))
|
|
|
|
|
|
def _row_references(row: dict[str, Any], fields: list[str], infer_fields: bool) -> list[dict[str, str]]:
|
|
refs: list[dict[str, str]] = []
|
|
for field in fields:
|
|
for value in _flatten_strings(_get_path_value(row, field)):
|
|
if pathlib.PurePosixPath(value).suffix.lower() in MEDIA_SUFFIXES:
|
|
refs.append({"field": field, "path": value})
|
|
if infer_fields:
|
|
for key, value in row.items():
|
|
if not _looks_like_media_field(str(key)):
|
|
continue
|
|
for candidate in _flatten_strings(value):
|
|
if pathlib.PurePosixPath(candidate).suffix.lower() in MEDIA_SUFFIXES:
|
|
refs.append({"field": str(key), "path": candidate})
|
|
seen: set[tuple[str, str]] = set()
|
|
unique: list[dict[str, str]] = []
|
|
for ref in refs:
|
|
key = (ref["field"], ref["path"])
|
|
if key in seen:
|
|
continue
|
|
unique.append(ref)
|
|
seen.add(key)
|
|
return unique
|
|
|
|
|
|
def _resolve_path(root: pathlib.Path, value: str) -> pathlib.Path:
|
|
raw = str(value).strip()
|
|
path = pathlib.Path(raw)
|
|
if path.is_absolute():
|
|
return path
|
|
for base in (root, root / "data", root / "images", root / "videos", root / "media"):
|
|
candidate = base / path
|
|
if candidate.exists():
|
|
return candidate
|
|
return root / path
|
|
|
|
|
|
def _inspect_file(path: pathlib.Path, helpers: HelperMap, deep_check: bool) -> dict[str, Any]:
|
|
suffix = path.suffix.lower()
|
|
payload: dict[str, Any] = {"path": str(path), "exists": path.exists(), "suffix": suffix, "readable": False}
|
|
if not path.exists() or not path.is_file():
|
|
return payload
|
|
payload["sizeBytes"] = path.stat().st_size
|
|
if not deep_check:
|
|
payload["readable"] = True
|
|
return payload
|
|
if suffix in IMAGE_SUFFIXES:
|
|
image = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
|
payload["readable"] = image is not None
|
|
if image is not None:
|
|
payload["shape"] = [int(item) for item in image.shape]
|
|
return payload
|
|
if suffix in DEPTH_SUFFIXES:
|
|
if suffix == ".npy":
|
|
data = np.load(path)
|
|
payload["readable"] = True
|
|
payload["shape"] = [int(item) for item in data.shape]
|
|
payload["dtype"] = str(data.dtype)
|
|
else:
|
|
image = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
|
payload["readable"] = image is not None
|
|
if image is not None:
|
|
payload["shape"] = [int(item) for item in image.shape]
|
|
return payload
|
|
if suffix in VIDEO_SUFFIXES:
|
|
try:
|
|
metrics = helpers["inspect_video_file"](path)
|
|
payload["readable"] = int(metrics.get("frameCount") or 0) > 0
|
|
payload["video"] = metrics
|
|
except Exception as exc:
|
|
payload["error"] = str(exc)
|
|
return payload
|
|
if suffix in POINTCLOUD_SUFFIXES:
|
|
with path.open("rb") as handle:
|
|
head = handle.read(128)
|
|
payload["readable"] = bool(head)
|
|
payload["headerPreview"] = head[:64].decode("utf-8", errors="replace")
|
|
return payload
|
|
payload["readable"] = True
|
|
return payload
|
|
|
|
|
|
def run(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
source_path = helpers["resolve_source_path"](params, inputs)
|
|
if source_path is None:
|
|
raise RuntimeError("媒体引用可读性校验需要 SOURCE_PATH 指向真实 manifest 或数据集目录")
|
|
max_files = _int_param(params, "MAX_FILES", 50)
|
|
max_rows = _int_param(params, "MAX_ROWS", 200000)
|
|
max_references = _int_param(params, "MAX_REFERENCES", 5000)
|
|
deep_check = _bool_param(params, "DEEP_CHECK", True)
|
|
infer_fields = _bool_param(params, "INFER_FIELDS", True)
|
|
fields = _field_list(params.get("MEDIA_FIELDS", ""), DEFAULT_MEDIA_FIELDS)
|
|
root = source_path if source_path.is_dir() else source_path.parent
|
|
files = _candidate_files(source_path, max_files)
|
|
if not files:
|
|
raise RuntimeError(f"{source_path} does not contain readable media manifest CSV/JSON/JSONL/Parquet files")
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
parsed_files: list[str] = []
|
|
for file_path in files:
|
|
if len(rows) >= max_rows:
|
|
break
|
|
next_rows = _rows_from_file(file_path, max(1, max_rows - len(rows)))
|
|
if next_rows:
|
|
rows.extend(next_rows)
|
|
parsed_files.append(str(file_path))
|
|
if not rows:
|
|
raise RuntimeError(f"{source_path} media manifest files are empty")
|
|
|
|
references: list[dict[str, Any]] = []
|
|
field_counts: dict[str, int] = {}
|
|
missing: list[dict[str, Any]] = []
|
|
unreadable: list[dict[str, Any]] = []
|
|
inspected_cache: dict[str, dict[str, Any]] = {}
|
|
for row_index, row in enumerate(rows):
|
|
for ref in _row_references(row, fields, infer_fields):
|
|
if len(references) >= max_references:
|
|
break
|
|
resolved = _resolve_path(root, ref["path"])
|
|
cache_key = str(resolved)
|
|
inspected = inspected_cache.get(cache_key)
|
|
if inspected is None:
|
|
inspected = _inspect_file(resolved, helpers, deep_check)
|
|
inspected_cache[cache_key] = inspected
|
|
item = {
|
|
"rowIndex": row_index,
|
|
"field": ref["field"],
|
|
"reference": ref["path"],
|
|
"resolvedPath": str(resolved),
|
|
"exists": inspected.get("exists", False),
|
|
"readable": inspected.get("readable", False),
|
|
"suffix": inspected.get("suffix", ""),
|
|
"sizeBytes": inspected.get("sizeBytes", 0),
|
|
}
|
|
references.append(item)
|
|
field_counts[ref["field"]] = field_counts.get(ref["field"], 0) + 1
|
|
if not item["exists"]:
|
|
missing.append(item)
|
|
elif not item["readable"]:
|
|
unreadable.append(item)
|
|
if len(references) >= max_references:
|
|
break
|
|
|
|
issues: list[dict[str, Any]] = []
|
|
warnings: list[dict[str, Any]] = []
|
|
if not references:
|
|
issues.append({"code": "no_media_references_found", "fields": fields})
|
|
if missing:
|
|
issues.append({"code": "missing_media_files", "count": len(missing), "examples": missing[:100]})
|
|
if unreadable:
|
|
issues.append({"code": "unreadable_media_files", "count": len(unreadable), "examples": unreadable[:100]})
|
|
if len(references) >= max_references:
|
|
warnings.append({"code": "reference_scan_truncated", "maxReferences": max_references})
|
|
|
|
payload = {
|
|
"datasetType": "media-reference-validation",
|
|
"sourcePath": str(source_path),
|
|
"valid": not issues,
|
|
"issueCount": len(issues),
|
|
"warningCount": len(warnings),
|
|
"issues": issues,
|
|
"warnings": warnings,
|
|
"parsedFiles": parsed_files,
|
|
"rowCount": len(rows),
|
|
"referenceCount": len(references),
|
|
"uniqueResolvedPathCount": len(inspected_cache),
|
|
"missingCount": len(missing),
|
|
"unreadableCount": len(unreadable),
|
|
"fieldCounts": field_counts,
|
|
"references": references[:500],
|
|
}
|
|
return [
|
|
helpers["write_json_artifact"](
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"media-reference-validation.json",
|
|
payload,
|
|
output=helpers["output_binding"](node_data),
|
|
)
|
|
]
|