eust-w 43326d6a5e feat(platform): land workflow, data tooling, and docs updates
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
2026-04-24 12:29:46 +08:00

4683 lines
190 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
from __future__ import annotations
import datetime as dt
import contextlib
import fnmatch
import hashlib
import json
import mimetypes
import os
import pathlib
import base64
import csv
import math
import re
import resource
import shutil
import sqlite3
import subprocess
import sys
import tarfile
import time
import zipfile
from typing import Any
from urllib import error, request
from urllib.parse import urlparse
RESULT_PREFIX = "DROBOTICFLOW_RESULT="
RESULT_PATH_PREFIX = "DROBOTICFLOW_RESULT_PATH="
TEXT_FILE_SUFFIXES = {
".txt",
".log",
".md",
".yaml",
".yml",
".xml",
".html",
".htm",
}
TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"}
IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"}
ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"}
ARCHIVE_FILE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip")
XPLATFORM_VIDEO_META_FILES = ("video_meta.json", "video_metadata.json")
XPLATFORM_INLINE_JSON_FILES = {"meta.json", "intrinsics.json", *XPLATFORM_VIDEO_META_FILES}
PREVIEW_TEXT_LIMIT = 12_000
PREVIEW_TEXT_LINE_LIMIT = 240
PREVIEW_TABLE_ROW_LIMIT = 24
PREVIEW_FILE_LIST_LIMIT = 40
PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024
DATASET_PARTITION_SIZE = 500
QUALITY_SAMPLE_LIMIT = 48
QUALITY_ANALYSIS_MAX_DIMENSION = 640
QUALITY_SKIN_COMPONENT_MIN_RATIO = 0.002
RESERVED_MONGO_COLLECTIONS = {
"projects",
"operators",
"workflows",
"templates",
"runs",
"sourceStates",
"auditTasks",
"samplingTasks",
"users",
"storageConnections",
"settings",
}
DATASET_PARTITION_KEYS = (
"items",
"entries",
"rows",
"collections",
"catalogCollections",
"catalogItems",
"catalogEntries",
)
PAYLOAD_FILE_READ_ATTEMPTS = 60
PAYLOAD_FILE_READ_DELAY_SECONDS = 1.0
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def log(message: str) -> None:
print(message, flush=True)
def load_json_env(name: str, default: Any) -> Any:
value = os.getenv(name, "").strip()
if not value:
return default
return json.loads(value)
def load_json_file(path: str, default: Any) -> Any:
raw = str(path or "").strip()
if not raw:
return default
candidate = pathlib.Path(raw)
if not candidate.exists():
return default
return json.loads(candidate.read_text(encoding="utf-8"))
def write_json_file(path: str | pathlib.Path, value: Any) -> pathlib.Path:
candidate = pathlib.Path(path)
candidate.parent.mkdir(parents=True, exist_ok=True)
candidate.write_text(json.dumps(json_compatible(value), ensure_ascii=False), encoding="utf-8")
return candidate
def text_value(value: Any) -> str:
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
return str(value or "")
def json_compatible(value: Any) -> Any:
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
if isinstance(value, dict):
return {str(key): json_compatible(item) for key, item in value.items()}
if isinstance(value, list):
return [json_compatible(item) for item in value]
if isinstance(value, tuple):
return [json_compatible(item) for item in value]
return value
def load_input_artifacts() -> list[dict[str, Any]]:
input_artifacts_path = os.getenv("INPUT_ARTIFACTS_PATH", "").strip()
if input_artifacts_path:
loaded = load_json_file(input_artifacts_path, [])
if isinstance(loaded, list):
return json_compatible(loaded)
loaded = load_json_env("INPUT_ARTIFACTS_JSON", [])
if isinstance(loaded, list):
return json_compatible(loaded)
return []
def stage_manifest_payload(
run_id: str,
workflow_id: str,
node_id: str,
node_label: str,
output_dir: pathlib.Path,
artifacts: list[dict[str, Any]],
) -> dict[str, Any]:
manifest_artifacts: list[dict[str, Any]] = []
data_refs: list[dict[str, Any]] = []
for artifact in artifacts:
item = {
"id": text_value(artifact.get("id")),
"name": text_value(artifact.get("name")),
"type": text_value(artifact.get("type")),
"path": text_value(artifact.get("path")),
"size": text_value(artifact.get("size")),
}
if text_value(artifact.get("portId")):
item["portId"] = text_value(artifact.get("portId"))
if text_value(artifact.get("portName")):
item["portName"] = text_value(artifact.get("portName"))
if isinstance(artifact.get("metadata"), dict) and artifact["metadata"]:
item["metadata"] = json_compatible(artifact["metadata"])
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
if data_ref:
item["dataRef"] = json_compatible(data_ref)
data_refs.append(
{
"artifactId": item["id"],
"artifactName": item["name"],
"portId": item.get("portId", ""),
**json_compatible(data_ref),
}
)
manifest_artifacts.append(item)
return {
"runId": run_id,
"workflowId": workflow_id,
"nodeId": node_id,
"nodeLabel": node_label,
"outputDir": str(output_dir),
"artifactCount": len(manifest_artifacts),
"artifacts": manifest_artifacts,
"dataRefs": data_refs,
"primaryDataRefs": data_refs,
"generatedAt": now_iso(),
}
def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path:
return output_dir.parent / "result.json"
def default_stage_manifest_path(output_dir: pathlib.Path) -> pathlib.Path:
return output_dir.parent / "stage-manifest.json"
def normalize_json_payload(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return json_compatible(value)
if isinstance(value, list):
return {"items": json_compatible(value), "itemCount": len(value)}
return {"value": json_compatible(value)}
def bos_prefix(value: str) -> str:
return str(value or "").strip().strip("/")
def canonical_bos_path(bucket: str, key: str) -> str:
normalized_bucket = str(bucket or "").strip()
normalized_key = str(key or "").strip().lstrip("/")
if not normalized_bucket:
return ""
if not normalized_key:
return f"bos://{normalized_bucket}"
return f"bos://{normalized_bucket}/{normalized_key}"
def parse_bos_path(value: str) -> tuple[str, str]:
raw = str(value or "").strip()
if not raw.startswith("bos://"):
return "", ""
bucket_and_key = raw[len("bos://") :]
if not bucket_and_key:
return "", ""
bucket, _, key = bucket_and_key.partition("/")
return bucket.strip(), bos_prefix(key)
def is_xplatform_video_meta_name(value: str) -> bool:
return str(value or "").strip().lower() in XPLATFORM_VIDEO_META_FILES
def load_collection_video_meta(collection_dir: pathlib.Path) -> tuple[dict[str, dict[str, Any]], str]:
for candidate in XPLATFORM_VIDEO_META_FILES:
payload = optional_json_file(collection_dir / candidate)
if payload is not None:
return parse_video_meta_rows(payload), candidate
return {}, ""
def sampled_frame_positions(frame_count: int, sample_limit: int = QUALITY_SAMPLE_LIMIT) -> list[int]:
if frame_count <= 0:
return []
if frame_count <= sample_limit:
return list(range(frame_count))
last_frame = max(frame_count - 1, 0)
positions = {
min(last_frame, max(0, int(round(index * last_frame / max(sample_limit - 1, 1)))))
for index in range(sample_limit)
}
return sorted(positions)
def resize_quality_frame(frame: Any, max_dimension: int = QUALITY_ANALYSIS_MAX_DIMENSION) -> Any:
try:
height, width = frame.shape[:2]
except Exception:
return frame
longest_edge = max(width, height)
if longest_edge <= 0 or longest_edge <= max_dimension:
return frame
scale = max_dimension / float(longest_edge)
target_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
cv2, _ = require_cv2()
return cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA)
def build_skin_mask(frame: Any) -> Any:
cv2, np = require_cv2()
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
ycrcb_mask = cv2.inRange(
ycrcb,
np.array((0, 133, 77), dtype=np.uint8),
np.array((255, 173, 127), dtype=np.uint8),
)
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
hsv_mask = cv2.inRange(
hsv,
np.array((0, 20, 40), dtype=np.uint8),
np.array((25, 220, 255), dtype=np.uint8),
)
return cv2.bitwise_and(ycrcb_mask, hsv_mask)
def largest_skin_component_ratio(mask: Any, min_component_ratio: float = QUALITY_SKIN_COMPONENT_MIN_RATIO) -> float:
cv2, np = require_cv2()
if mask is None:
return 0.0
total_area = int(mask.shape[0] * mask.shape[1]) if len(mask.shape) >= 2 else 0
if total_area <= 0:
return 0.0
kernel = np.ones((5, 5), dtype=np.uint8)
cleaned = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_CLOSE, kernel)
component_threshold = max(1, int(total_area * max(min_component_ratio, 0.0)))
count, _, stats, _ = cv2.connectedComponentsWithStats(cleaned)
largest_area = 0
for index in range(1, count):
area = int(stats[index, cv2.CC_STAT_AREA])
if area < component_threshold:
continue
if area > largest_area:
largest_area = area
return float(largest_area / total_area)
def motion_jitter_ratio(shifts: list[tuple[float, float]], diagonal: float) -> float:
if not shifts:
return 0.0
_, np = require_cv2()
normalized_diagonal = max(diagonal, 1.0)
dx_values = [item[0] for item in shifts]
dy_values = [item[1] for item in shifts]
median_dx = float(np.median(dx_values))
median_dy = float(np.median(dy_values))
jitters = [
math.sqrt((dx - median_dx) ** 2 + (dy - median_dy) ** 2) / normalized_diagonal
for dx, dy in shifts
]
return sum(jitters) / max(len(jitters), 1)
def bos_key_join(prefix: str, relative_path: str) -> str:
normalized_prefix = bos_prefix(prefix)
normalized_relative = str(relative_path or "").strip().lstrip("/")
if not normalized_prefix:
return normalized_relative
if not normalized_relative:
return normalized_prefix
return f"{normalized_prefix}/{normalized_relative}"
def require_bos_sdk():
try:
import baidubce.protocol as protocol # type: ignore
from baidubce.auth.bce_credentials import BceCredentials # type: ignore
from baidubce.bce_client_configuration import BceClientConfiguration # type: ignore
from baidubce.services.bos.bos_client import BosClient # type: ignore
except Exception as error: # pragma: no cover - runtime environment specific
raise RuntimeError("BOS 扫描需要 bce-python-sdk 运行时") from error
return protocol, BceCredentials, BceClientConfiguration, BosClient
def build_bos_client(endpoint: str, access_key_id: str, secret_access_key: str):
protocol_module, BceCredentials, BceClientConfiguration, BosClient = require_bos_sdk()
parsed = urlparse(endpoint if "://" in endpoint else f"https://{endpoint}")
scheme = (parsed.scheme or "https").lower()
host = parsed.netloc or parsed.path
if not host:
raise RuntimeError("BOS endpoint 不能为空")
config = BceClientConfiguration(
credentials=BceCredentials(access_key_id, secret_access_key),
endpoint=host,
protocol=protocol_module.HTTPS if scheme == "https" else protocol_module.HTTP,
)
return BosClient(config), scheme
def resource_snapshot() -> tuple[float, int]:
usage = resource.getrusage(resource.RUSAGE_SELF)
cpu_seconds = usage.ru_utime + usage.ru_stime
if sys.platform == "darwin":
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
else:
memory_mib = max(1, int(usage.ru_maxrss / 1024))
return cpu_seconds, memory_mib
def resource_series(start_cpu: float, start_memory: int, started_at: str) -> list[dict[str, Any]]:
end_cpu, end_memory = resource_snapshot()
ended_at = now_iso()
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
elapsed = max((ended - started).total_seconds(), 0.001)
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
return [
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, "gpu": 0},
]
def format_size(num_bytes: int) -> str:
if num_bytes < 1024:
return f"{num_bytes} B"
if num_bytes < 1024 * 1024:
return f"{num_bytes / 1024:.1f} KiB"
if num_bytes < 1024 * 1024 * 1024:
return f"{num_bytes / (1024 * 1024):.1f} MiB"
return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB"
MAX_SHA256_FILE_BYTES = 64 * 1024 * 1024
def sha256_file(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(65536), b""):
digest.update(chunk)
return digest.hexdigest()
def artifact_sha256(path: pathlib.Path, size_bytes: int) -> tuple[str, dict[str, Any]]:
if size_bytes > MAX_SHA256_FILE_BYTES:
return "", {"sha256Skipped": True, "sha256SkipReason": "size-limit", "sha256SkipBytes": size_bytes}
return sha256_file(path), {}
def require_cv2() -> tuple[Any, Any]:
try:
import cv2 # type: ignore
import numpy as np # type: ignore
except Exception as error: # pragma: no cover - environment specific
raise RuntimeError("视频算子需要 opencv-python-headless 和 numpy 运行时") from error
return cv2, np
def data_url_for_path(path: pathlib.Path, mime_type: str) -> str:
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f"data:{mime_type};base64,{encoded}"
def decode_data_url(url: str) -> tuple[str, bytes] | None:
if not url.startswith("data:") or "," not in url:
return None
header, encoded = url.split(",", 1)
if ";base64" not in header:
return None
mime_type = header[5:].split(";", 1)[0] or "application/octet-stream"
try:
return mime_type, base64.b64decode(encoded)
except Exception:
return None
def download_remote_source_to_path(url: str, path: pathlib.Path) -> pathlib.Path | None:
parsed = urlparse(url)
if parsed.scheme.lower() not in {"http", "https"}:
return None
path.parent.mkdir(parents=True, exist_ok=True)
remote_request = request.Request(
url,
headers={
"User-Agent": "DroboticFlow operator-runner/1.0",
},
)
try:
with request.urlopen(remote_request, timeout=300) as response, path.open("wb") as handle:
shutil.copyfileobj(response, handle, length=1024 * 1024)
except error.HTTPError as exc:
raise RuntimeError(f"下载远程输入失败({exc.code}): {path.name}") from exc
except error.URLError as exc:
raise RuntimeError(f"下载远程输入失败: {path.name}") from exc
return path
def download_remote_source(url: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
path = output_dir / "_materialized_sources" / default_name
return download_remote_source_to_path(url, path)
def workspace_cached_bos_source_path(raw_path: str, output_dir: pathlib.Path) -> pathlib.Path | None:
bucket, key = parse_bos_path(raw_path)
if not bucket or not key:
return None
workspace_root = str(os.getenv("WORKSPACE_ROOT", "")).strip()
if workspace_root:
root = pathlib.Path(workspace_root)
if root.is_absolute():
return root / "_bos_cache" / bucket / key
return output_dir / "_materialized_sources" / bucket / key
def download_bos_source(raw_path: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
target = workspace_cached_bos_source_path(raw_path, output_dir)
if target is None:
return None
if target.exists() and target.stat().st_size > 0:
return target
bucket, key = parse_bos_path(raw_path)
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
if not bucket or not key or not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
return None
target.parent.mkdir(parents=True, exist_ok=True)
temp_target = target.with_name(target.name + ".part")
with contextlib.suppress(FileNotFoundError):
temp_target.unlink()
try:
client, _ = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
client.get_object_to_file(bucket, key, str(temp_target))
temp_target.replace(target)
except Exception as exc:
with contextlib.suppress(FileNotFoundError):
temp_target.unlink()
raise RuntimeError(f"下载 BOS 输入失败: {default_name}") from exc
return target
def signed_bos_source_url(raw_path: str) -> str:
bucket, key = parse_bos_path(raw_path)
if not bucket or not key:
return ""
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
if not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
return ""
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
return text_value(
client.generate_pre_signed_url(
bucket,
key,
expiration_in_seconds=3600,
protocol=scheme,
)
).strip()
def remote_video_url_for_source(source: dict[str, Any]) -> str:
preview_url = str(source.get("previewURL", "")).strip()
parsed_preview = urlparse(preview_url)
if parsed_preview.scheme.lower() in {"http", "https"}:
return preview_url
raw_path = str(source.get("path", "")).strip()
if raw_path.startswith("bos://"):
return signed_bos_source_url(raw_path)
return ""
def is_archive_path(path: pathlib.Path) -> bool:
name = path.name.lower()
return any(name.endswith(suffix) for suffix in ARCHIVE_FILE_SUFFIXES)
def safe_extract_archive(path: pathlib.Path, destination: pathlib.Path) -> list[str]:
destination.mkdir(parents=True, exist_ok=True)
destination_resolved = destination.resolve()
extracted_files: list[str] = []
def ensure_child(member_name: str) -> None:
target = (destination / member_name).resolve()
if target != destination_resolved and destination_resolved not in target.parents:
raise RuntimeError(f"archive entry escapes target directory: {member_name}")
if path.name.lower().endswith(".zip"):
with zipfile.ZipFile(path) as archive:
for member in archive.infolist():
if not member.filename or member.is_dir():
continue
ensure_child(member.filename)
extracted_files.append(member.filename)
archive.extractall(destination)
return extracted_files
with tarfile.open(path, "r:*") as archive:
members = archive.getmembers()
for member in members:
if not member.name or member.isdir():
continue
ensure_child(member.name)
extracted_files.append(member.name)
archive.extractall(destination)
return extracted_files
def artifact_slug(value: str) -> str:
normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-")
return normalized or "artifact"
def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str:
suffix = path.suffix.lower()
if suffix == ".json":
return "json"
if suffix in VIDEO_FILE_SUFFIXES:
return "video"
if suffix in IMAGE_FILE_SUFFIXES:
return "image"
return fallback
def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None:
cleaned = {key: json_compatible(value) for key, value in metadata.items() if value not in ("", None, [], {})}
return cleaned or None
def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None:
if path.suffix.lower() != ".json":
return None
for attempt in range(PAYLOAD_FILE_READ_ATTEMPTS):
if not path.exists():
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
continue
return None
try:
with path.open("r", encoding="utf-8") as handle:
value = json.load(handle)
except (FileNotFoundError, OSError, json.JSONDecodeError):
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
continue
return None
except Exception:
return None
if isinstance(value, dict):
return value
if isinstance(value, list):
return {"items": value, "itemCount": len(value)}
return {"value": value}
return None
def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None:
outputs = node_data.get("outputs", []) or []
if name:
for output in outputs:
if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name:
return output
if 0 <= index < len(outputs):
return outputs[index]
return None
def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]:
with path.open("rb") as handle:
raw = handle.read(PREVIEW_TEXT_LIMIT + 1)
truncated = len(raw) > PREVIEW_TEXT_LIMIT
text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace")
lines = text.splitlines()
if len(lines) > PREVIEW_TEXT_LINE_LIMIT:
text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT])
truncated = True
line_count = max(1, text.count("\n") + 1) if text else 0
return text, truncated, line_count
def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]:
text, truncated, line_count = read_text_sample(path)
return {
"preview": {
"kind": "text",
"format": format_name,
"text": text,
"truncated": truncated,
"lineCount": line_count,
}
}
def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]:
columns: list[str] = []
rows: list[dict[str, Any]] = []
total_rows = 0
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
reader = csv.reader(handle, delimiter=delimiter)
for index, row in enumerate(reader):
if index == 0:
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
continue
total_rows += 1
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
continue
width = max(len(columns), len(row))
row_payload: dict[str, Any] = {}
for column_index in range(width):
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
rows.append(row_payload)
return {
"preview": {
"kind": "table",
"format": format_name,
"columns": columns,
"rows": rows,
"rowCount": total_rows,
"truncated": total_rows > len(rows),
}
}
def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]:
columns: list[str] = []
rows: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
reader = csv.reader(handle, delimiter=delimiter)
for index, row in enumerate(reader):
if index == 0:
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
continue
width = max(len(columns), len(row))
row_payload: dict[str, Any] = {}
for column_index in range(width):
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
rows.append(row_payload)
return columns, rows
def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]:
rows: list[Any] = []
line_count = 0
with path.open("r", encoding="utf-8", errors="replace") as handle:
for raw_line in handle:
line = raw_line.strip()
if not line:
continue
line_count += 1
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
continue
try:
rows.append(json.loads(line))
except Exception:
rows.append({"value": line})
columns: list[str] = []
for row in rows:
if isinstance(row, dict):
for key in row.keys():
if key not in columns:
columns.append(key)
return {
"preview": {
"kind": "table",
"format": "jsonl",
"columns": columns,
"rows": rows,
"rowCount": line_count,
"truncated": line_count > len(rows),
}
}
def inspect_video_file(path: pathlib.Path) -> dict[str, Any]:
cv2, _ = require_cv2()
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
return {}
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
payload = clean_metadata(
{
"frameRate": round(fps, 3) if fps > 0 else None,
"frameCount": frame_count or None,
"resolution": f"{width}x{height}" if width > 0 and height > 0 else None,
"duration": round(duration, 3) if duration > 0 else None,
}
)
return payload or {}
finally:
capture.release()
def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]:
payload: dict[str, Any] = {
"preview": {
"kind": "image",
"format": path.suffix.lower().lstrip(".") or "image",
}
}
size_bytes = path.stat().st_size if path.exists() else 0
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
payload["preview"]["url"] = data_url_for_path(path, mime_type)
return payload
def preview_media_payload(path: pathlib.Path, mime_type: str, kind: str) -> dict[str, Any]:
preview: dict[str, Any] = {
"kind": kind,
"format": path.suffix.lower().lstrip(".") or kind,
}
if kind == "video":
preview.update(inspect_video_file(path))
size_bytes = path.stat().st_size if path.exists() else 0
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
preview["url"] = data_url_for_path(path, mime_type)
return {"preview": preview}
def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]:
try:
relative = candidate.relative_to(root).as_posix()
except ValueError:
relative = candidate.name
items.append({"path": relative, "sizeBytes": candidate.stat().st_size})
return items
def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]:
files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()]
extensions: dict[str, int] = {}
total_bytes = 0
for candidate in files:
extension = candidate.suffix.lower() or "<none>"
extensions[extension] = extensions.get(extension, 0) + 1
total_bytes += candidate.stat().st_size
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
return {
"root": str(root),
"fileCount": len(files),
"totalBytes": total_bytes,
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
"files": sample_file_entries(files, root),
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
}
def optional_json_file(path: pathlib.Path) -> Any:
if not path.exists():
return None
try:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
except Exception:
return None
def optional_jsonl_rows(path: pathlib.Path) -> list[Any]:
rows: list[Any] = []
if not path.exists():
return rows
try:
with path.open("r", encoding="utf-8", errors="replace") as handle:
for raw_line in handle:
line = raw_line.strip()
if not line:
continue
rows.append(json.loads(line))
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
break
except Exception:
return []
return rows
def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None:
candidates = [path] if path.is_dir() else [path.parent]
candidates.extend(list((path if path.is_dir() else path.parent).parents))
for candidate in candidates:
meta_dir = candidate / "meta"
if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")):
return candidate
return None
def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]:
tree = summarize_directory_tree(root)
meta_dir = root / "meta"
info = optional_json_file(meta_dir / "info.json")
stats = optional_json_file(meta_dir / "stats.json")
tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl")
episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl")
parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()]
video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
return {
"preview": {
"kind": "dataset",
"format": "lerobot",
"root": str(root),
"summary": tree,
"info": info,
"stats": stats,
"tasks": tasks,
"episodes": episodes,
"parquetFiles": sample_file_entries(parquet_files, root),
"parquetCount": len(parquet_files),
"videoCount": len(video_files),
}
}
def sqlite_table_names(connection: sqlite3.Connection) -> list[str]:
rows = connection.execute(
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
).fetchall()
return [str(row[0]) for row in rows]
def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]:
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
try:
tables = sqlite_table_names(connection)
table_summaries: list[dict[str, Any]] = []
for table_name in tables[:12]:
count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0]
table_summaries.append({"name": table_name, "rowCount": int(count)})
payload: dict[str, Any] = {
"preview": {
"kind": "dataset",
"format": "sqlite",
"tables": table_summaries,
"tableCount": len(tables),
}
}
if "topics" in tables and "messages" in tables:
topic_rows = connection.execute(
"select id, name, type, serialization_format from topics order by id"
).fetchall()
message_counts = {
int(topic_id): int(count)
for topic_id, count in connection.execute(
"select topic_id, count(*) from messages group by topic_id order by topic_id"
).fetchall()
}
payload["preview"]["format"] = "rosbag2"
payload["preview"]["topics"] = [
{
"id": int(topic_id),
"name": str(name),
"type": str(topic_type),
"serialization": str(serialization),
"messageCount": message_counts.get(int(topic_id), 0),
}
for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT]
]
payload["preview"]["messageCount"] = int(sum(message_counts.values()))
return payload
finally:
connection.close()
def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]:
suffix = path.suffix.lower()
if suffix == ".db3":
return summarize_sqlite_payload(path)
header = ""
with path.open("rb") as handle:
header = handle.read(32).decode("utf-8", errors="replace")
format_name = "mcap" if suffix == ".mcap" else "rosbag1"
return {
"preview": {
"kind": "dataset",
"format": format_name,
"header": header.strip(),
"note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。",
}
}
def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None:
direct = params.get("SOURCE_PATH", "").strip()
if direct and pathlib.Path(direct).exists():
return pathlib.Path(direct)
for artifact in inputs:
payload = artifact.get("data")
if isinstance(payload, dict):
candidate = payload.get("sourcePath")
if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists():
return pathlib.Path(candidate)
source_path = artifact.get("path")
if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists():
return pathlib.Path(source_path)
return None
def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None:
data = artifact.get("data")
if not isinstance(data, dict):
return None
preview = data.get("preview")
if isinstance(preview, dict):
return preview
return None
def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]:
extensions: dict[str, int] = {}
files: list[dict[str, Any]] = []
for artifact in inputs:
name = str(artifact.get("name", "") or artifact.get("path", "")).strip()
extension = pathlib.Path(name).suffix.lower() or "<none>"
extensions[extension] = extensions.get(extension, 0) + 1
files.append(
{
"path": str(artifact.get("path", "")).strip() or name,
"name": name,
"size": str(artifact.get("size", "")),
}
)
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
return {
"fileCount": len(files),
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
"files": files[:PREVIEW_FILE_LIST_LIMIT],
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
}
def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str:
for artifact in inputs:
raw_path = str(artifact.get("path", "")).strip()
if not raw_path:
continue
parts = pathlib.PurePosixPath(raw_path).parts
for marker in ("meta", "data", "videos"):
if marker in parts:
index = parts.index(marker)
if index > 0:
return str(pathlib.PurePosixPath(*parts[:index]))
return ""
def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]:
info = None
stats = None
tasks: list[Any] = []
episodes: list[Any] = []
parquet_files: list[dict[str, Any]] = []
video_count = 0
for artifact in inputs:
name = str(artifact.get("name", "")).strip()
path = str(artifact.get("path", "")).strip()
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
preview = artifact_preview_payload(artifact) or {}
if name == "info.json" and isinstance(data, dict):
info = data
elif name == "stats.json" and isinstance(data, dict):
stats = data
elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list):
tasks = preview["rows"]
elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list):
episodes = preview["rows"]
if name.endswith(".parquet"):
parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))})
if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video":
video_count += 1
return {
"preview": {
"kind": "dataset",
"format": "lerobot",
"root": infer_lerobot_root_from_inputs(inputs),
"summary": summarize_input_artifacts(inputs),
"info": info,
"stats": stats,
"tasks": tasks,
"episodes": episodes,
"parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT],
"parquetCount": len(parquet_files),
"videoCount": video_count,
}
}
def parse_float_param(params: dict[str, str], key: str, default: float) -> float:
raw = params.get(key, "").strip()
if not raw:
return default
try:
return float(raw)
except Exception:
return default
def float_value(value: Any, default: float) -> float:
if value is None:
return default
if isinstance(value, str) and not value.strip():
return default
try:
return float(value)
except Exception:
return default
def parse_int_param(params: dict[str, str], key: str, default: int) -> int:
raw = params.get(key, "").strip()
if not raw:
return default
try:
return int(raw)
except Exception:
return default
def parse_bool_param(params: dict[str, str], key: str, default: bool = False) -> bool:
raw = params.get(key, "").strip().lower()
if not raw:
return default
return raw in {"1", "true", "yes", "on"}
def first_payload_with_keys(inputs: list[dict[str, Any]], *keys: str) -> dict[str, Any] | None:
for artifact in inputs:
payload = load_artifact_payload(artifact)
if isinstance(payload, dict) and all(key in payload for key in keys):
return payload
return None
def common_input_root(paths: list[str]) -> str:
filtered = [item for item in paths if item]
if not filtered:
return ""
if len(filtered) == 1:
return str(pathlib.Path(filtered[0]).parent)
try:
return os.path.commonpath(filtered)
except Exception:
return str(pathlib.Path(filtered[0]).parent)
def artifact_preview_url(artifact: dict[str, Any]) -> str:
preview = artifact_preview_payload(artifact) or {}
value = preview.get("url")
return value if isinstance(value, str) else ""
def source_ref_from_artifact(artifact: dict[str, Any], relative_path: str = "") -> dict[str, Any]:
metadata = artifact.get("metadata") if isinstance(artifact.get("metadata"), dict) else {}
return clean_metadata(
{
"name": str(artifact.get("name", "")).strip(),
"path": str(artifact.get("path", "")).strip(),
"relativePath": relative_path,
"previewURL": artifact_preview_url(artifact),
"mimeType": str(metadata.get("mimeType", "")).strip(),
}
) or {}
def materialize_artifact_to_path(artifact: dict[str, Any], destination: pathlib.Path) -> bool:
raw_path = str(artifact.get("path", "")).strip()
destination.parent.mkdir(parents=True, exist_ok=True)
if raw_path and pathlib.Path(raw_path).exists():
shutil.copy2(raw_path, destination)
return True
preview_url = artifact_preview_url(artifact)
decoded = decode_data_url(preview_url) if preview_url else None
if decoded is not None:
_, payload = decoded
destination.write_bytes(payload)
return True
payload = artifact.get("data")
if isinstance(payload, dict) and destination.suffix.lower() == ".json":
destination.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return True
preview = artifact_preview_payload(artifact) or {}
if preview.get("kind") == "text":
destination.write_text(str(preview.get("text", "")), encoding="utf-8")
return True
return False
def upload_relative_path(artifact: dict[str, Any], input_root: str, fallback_name: str) -> str:
explicit_relative_path = str(artifact.get("relativePath", "")).strip()
if explicit_relative_path:
return pathlib.PurePosixPath(explicit_relative_path).as_posix().lstrip("/")
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
payload_relative_path = str(payload.get("relativePath", "")).strip()
if payload_relative_path:
return pathlib.PurePosixPath(payload_relative_path).as_posix().lstrip("/")
raw_path = str(artifact.get("path", "")).strip()
relative_path = str(fallback_name or artifact.get("name", "")).strip() or "artifact"
if raw_path and input_root:
try:
relative_path = pathlib.Path(raw_path).relative_to(input_root).as_posix()
except Exception:
relative_path = pathlib.Path(raw_path).name or relative_path
if not relative_path:
relative_path = pathlib.Path(fallback_name or "artifact").name or "artifact"
return pathlib.PurePosixPath(relative_path).as_posix().lstrip("/")
def materialized_upload_source(artifact: dict[str, Any], staging_dir: pathlib.Path, relative_path: str) -> pathlib.Path | None:
raw_path = str(artifact.get("path", "")).strip()
if raw_path and os.path.exists(raw_path):
return pathlib.Path(raw_path)
destination = staging_dir / "_materialized_upload_inputs" / relative_path
if destination.exists():
return destination
if materialize_artifact_to_path(artifact, destination):
return destination
return None
def package_config_payload(item: dict[str, Any], manifest_payload: dict[str, Any]) -> dict[str, Any]:
return {
"collectionId": item.get("collectionId", ""),
"meta": item.get("meta", {}),
"segmentDurationSeconds": manifest_payload.get("segmentDurationSeconds"),
"sourceVideo": item.get("sourceVideo", ""),
"startSecond": item.get("startSecond"),
"endSecond": item.get("endSecond"),
"duration": item.get("duration") or item.get("clipDuration"),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
}
def package_related_artifact_names(manifest_payload: dict[str, Any]) -> set[str]:
names: set[str] = set()
for item in manifest_payload.get("items", []):
if not isinstance(item, dict):
continue
clip_id = str(item.get("clipId", "")).strip()
clip_name = str(item.get("clipName", "")).strip()
annotation_name = str(item.get("annotationFile", "")).strip() or (f"{clip_id}_annotation.json" if clip_id else "")
quality_name = f"{clip_id}_quality.json" if clip_id else ""
config_name = f"{clip_id}_config.json" if clip_id else ""
for candidate in (clip_name, annotation_name, quality_name, config_name):
if candidate:
names.add(candidate)
return names
def materialize_package_artifacts_from_manifest(
run_id: str,
node_id: str,
staging_dir: pathlib.Path,
manifest_payload: dict[str, Any],
node_data: dict[str, Any],
) -> list[dict[str, Any]]:
archive_sources = manifest_payload.get("archiveSources", {}) if isinstance(manifest_payload.get("archiveSources"), dict) else {}
binding = output_binding(node_data)
synthetic_artifacts: list[dict[str, Any]] = []
materialized_root = staging_dir / "_materialized_upload_inputs"
for item in manifest_payload.get("items", []):
if not isinstance(item, dict):
continue
clip_id = str(item.get("clipId", "")).strip()
if not clip_id:
continue
package_name = str(item.get("packageName", "")).strip() or f"{clip_id}_package"
clip_name = str(item.get("clipName", "")).strip() or f"{clip_id}.mp4"
package_relative_dir = pathlib.PurePosixPath("packages") / package_name
package_dir = materialized_root / package_relative_dir
package_dir.mkdir(parents=True, exist_ok=True)
clip_destination = package_dir / clip_name
direct_clip_path = str(item.get("clipPath", "")).strip()
clip_source: pathlib.Path | None = pathlib.Path(direct_clip_path) if direct_clip_path and pathlib.Path(direct_clip_path).exists() else None
if clip_source is None:
clip_source = materialize_clip_from_source(item, staging_dir, archive_sources)
if clip_source is not None and clip_source.exists():
clip_size = clip_source.stat().st_size
clip_mime_type = mimetypes.guess_type(clip_name)[0] or "video/mp4"
clip_artifact = {
"id": f"{run_id}-{node_id}-{package_name}-{clip_name}",
"runId": run_id,
"nodeId": node_id,
"name": clip_name,
"type": "video",
"size": format_size(clip_size),
"path": str(clip_source),
"metadata": {"mimeType": clip_mime_type},
"data": {
"relativePath": (package_relative_dir / clip_name).as_posix(),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
},
}
clip_artifact["relativePath"] = (package_relative_dir / clip_name).as_posix()
synthetic_artifacts.append(clip_artifact)
annotation_name = str(item.get("annotationFile", "")).strip() or f"{clip_id}_annotation.json"
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
annotation_artifact = write_json_artifact(run_id, node_id, package_dir, annotation_name, annotation_payload, output=binding)
annotation_artifact["relativePath"] = (package_relative_dir / annotation_name).as_posix()
synthetic_artifacts.append(annotation_artifact)
quality_name = f"{clip_id}_quality.json"
quality_payload = {
"clip_id": clip_id,
"source_video": item.get("sourceVideo", ""),
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
}
quality_artifact = write_json_artifact(run_id, node_id, package_dir, quality_name, quality_payload, output=binding)
quality_artifact["relativePath"] = (package_relative_dir / quality_name).as_posix()
synthetic_artifacts.append(quality_artifact)
config_name = f"{clip_id}_config.json"
config_artifact = write_json_artifact(
run_id,
node_id,
package_dir,
config_name,
package_config_payload(item, manifest_payload),
output=binding,
)
config_artifact["relativePath"] = (package_relative_dir / config_name).as_posix()
synthetic_artifacts.append(config_artifact)
return synthetic_artifacts
def materialize_input_tree(
inputs: list[dict[str, Any]],
output_dir: pathlib.Path,
) -> tuple[pathlib.Path | None, dict[str, dict[str, Any]]]:
paths = [str(item.get("path", "")).strip() for item in inputs if str(item.get("path", "")).strip()]
root = common_input_root(paths)
if not root:
return None, {}
target_root = output_dir / "_materialized_inputs"
if target_root.exists():
shutil.rmtree(target_root)
target_root.mkdir(parents=True, exist_ok=True)
source_refs: dict[str, dict[str, Any]] = {}
for index, artifact in enumerate(inputs):
raw_path = str(artifact.get("path", "")).strip()
relative_path = str(artifact.get("name", "")).strip() or f"artifact-{index}"
if raw_path:
try:
relative_path = pathlib.Path(raw_path).relative_to(root).as_posix()
except Exception:
relative_path = pathlib.Path(raw_path).name or relative_path
destination = target_root / relative_path
if materialize_artifact_to_path(artifact, destination):
source_refs[relative_path] = source_ref_from_artifact(artifact, relative_path)
if destination.is_file() and is_archive_path(destination):
extracted_files = safe_extract_archive(destination, target_root)
archive_preview_url = artifact_preview_url(artifact)
archive_key = artifact_slug(hashlib.sha256(archive_preview_url.encode("utf-8")).hexdigest()[:16]) if archive_preview_url else artifact_slug(relative_path)
for extracted_relative_path in extracted_files:
normalized_relative_path = pathlib.PurePosixPath(extracted_relative_path).as_posix().lstrip("/")
source_refs[normalized_relative_path] = clean_metadata(
{
"name": pathlib.PurePosixPath(normalized_relative_path).name,
"path": str(target_root / normalized_relative_path),
"relativePath": normalized_relative_path,
"archiveKey": archive_key,
"archivePreviewURL": archive_preview_url,
"archiveMemberPath": normalized_relative_path,
"archiveName": destination.name,
"mimeType": mimetypes.guess_type(normalized_relative_path)[0] or "",
}
) or {}
return target_root, source_refs
def detect_xplatform_collections(root: pathlib.Path) -> list[pathlib.Path]:
if not root.exists():
return []
candidates: list[pathlib.Path] = []
if (root / "meta.json").exists():
return [root]
for child in sorted(root.iterdir()):
if not child.is_dir():
continue
if (child / "meta.json").exists():
candidates.append(child)
continue
if any(candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES for candidate in child.rglob("*")):
candidates.append(child)
return candidates
def normalize_collection_relative_path(value: pathlib.PurePath | str) -> str:
text = value.as_posix() if isinstance(value, pathlib.PurePath) else str(value or "")
normalized = text.strip().strip("/")
if normalized in {"", "."}:
return ""
return normalized
def collection_name_from_source_path(source_path: str, fallback_name: str = "collection") -> str:
candidate = pathlib.PurePosixPath(str(source_path or "").rstrip("/")).name
return candidate or fallback_name
def parse_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
rows: dict[str, dict[str, Any]] = {}
if not isinstance(value, list):
return rows
for item in value:
if not isinstance(item, dict):
continue
filename = str(item.get("filename", "")).strip()
if not filename:
continue
rows[filename] = item
return rows
def collection_file_entry(
root: pathlib.Path,
collection_dir: pathlib.Path,
candidate: pathlib.Path,
collection_id: str,
meta: dict[str, Any],
video_meta_rows: dict[str, dict[str, Any]],
) -> dict[str, Any]:
relative_path = candidate.relative_to(root).as_posix()
collection_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
collection_relative_file_path = candidate.relative_to(collection_dir).as_posix()
mime_type = mimetypes.guess_type(candidate.name)[0] or ""
entry: dict[str, Any] = {
"collectionId": collection_id,
"collectionName": collection_dir.name,
"collectionPath": str(collection_dir),
"collectionRelativePath": collection_relative_path,
"meta": meta,
"relativePath": relative_path,
"path": str(candidate),
"sizeBytes": candidate.stat().st_size,
"mimeType": mime_type,
"source": {"path": str(candidate), "relativePath": relative_path, "mimeType": mime_type},
}
if candidate.suffix.lower() == ".json":
payload = optional_json_file(candidate)
if payload is not None:
entry["data"] = payload
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
matched_meta = video_meta_rows.get(candidate.name, {})
entry.update(
{
"videoFolder": collection_relative_file_path.split("/", 1)[0] if "/" in collection_relative_file_path else "",
"filename": candidate.name,
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
}
)
return entry
def collection_summary(
root: pathlib.Path,
collection_dir: pathlib.Path,
meta: dict[str, Any],
video_meta_rows: dict[str, dict[str, Any]],
) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
files = [candidate for candidate in sorted(collection_dir.rglob("*")) if candidate.is_file()]
videos = [candidate for candidate in files if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
total_bytes = sum(candidate.stat().st_size for candidate in files)
required = ["meta.json", "video_meta.json"]
missing_required = []
if not (collection_dir / "meta.json").exists():
missing_required.append("meta.json")
if not any((collection_dir / candidate).exists() for candidate in XPLATFORM_VIDEO_META_FILES):
missing_required.append("video_meta.json")
has_intrinsics = (collection_dir / "intrinsics.json").exists()
collection_root_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
collection_id = collection_key(collection_root_relative_path, collection_dir.name)
items: list[dict[str, Any]] = []
entries: list[dict[str, Any]] = []
for candidate in files:
entry = collection_file_entry(root, collection_dir, candidate, collection_id, meta, video_meta_rows)
entries.append(entry)
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
items.append(dict(entry))
summary = {
"id": collection_id,
"name": collection_dir.name,
"path": str(collection_dir),
"relativePath": collection_root_relative_path,
"meta": meta,
"hasIntrinsics": has_intrinsics,
"hasVideoMeta": bool(video_meta_rows),
"requiredFiles": required,
"missingRequiredFiles": missing_required,
"videoCount": len(videos),
"fileCount": len(files),
"totalBytes": total_bytes,
"valid": len(missing_required) == 0 and len(videos) > 0,
}
return summary, items, entries
def source_path_join(root: str, relative_path: str) -> str:
if not root:
return relative_path
return root.rstrip("/") + "/" + relative_path.lstrip("/")
def manifest_source_ref(entry: dict[str, Any], relative_path: str, source_path: str) -> dict[str, Any]:
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
storage_path = str(source.get("path", "") or entry.get("storagePath", "") or source_path_join(source_path, relative_path)).strip()
return clean_metadata(
{
"path": storage_path,
"relativePath": relative_path,
"previewURL": str(source.get("previewURL", "")).strip(),
"mimeType": str(entry.get("mimeType", "")).strip(),
}
) or {}
def manifest_entry_json_payload(entry: Any, source_path: str = "") -> Any:
if not isinstance(entry, dict):
return None
inline_data = entry.get("data")
if inline_data is not None:
return inline_data
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
candidates = [str(source.get("path", "")).strip(), str(entry.get("storagePath", "")).strip(), str(entry.get("path", "")).strip()]
for raw_candidate in candidates:
if not raw_candidate:
continue
if "://" in raw_candidate and not raw_candidate.startswith("file://"):
continue
candidate_text = raw_candidate[len("file://") :] if raw_candidate.startswith("file://") else raw_candidate
candidate_path = pathlib.Path(candidate_text)
if not candidate_path.is_absolute():
normalized_source_path = str(source_path or "").strip()
if normalized_source_path and "://" not in normalized_source_path:
candidate_path = pathlib.Path(normalized_source_path) / candidate_text
if candidate_path.is_file():
return optional_json_file(candidate_path)
return None
def manifest_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
if isinstance(value, dict) and isinstance(value.get("items"), list):
return parse_video_meta_rows(value.get("items"))
return parse_video_meta_rows(value)
def collection_key(relative_path: str, fallback_name: str = "collection") -> str:
normalized = relative_path.strip().strip("/")
if not normalized:
return artifact_slug(fallback_name)
return artifact_slug(normalized)
def fallback_manifest_collections(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
grouped: dict[str, list[dict[str, Any]]] = {}
for entry in entries:
relative_path = str(entry.get("path", "")).strip().lstrip("/")
if not relative_path:
continue
parts = pathlib.PurePosixPath(relative_path).parts
if len(parts) < 2:
continue
grouped.setdefault(parts[0], []).append({**entry, "path": relative_path})
collection_summaries: list[dict[str, Any]] = []
items: list[dict[str, Any]] = []
indexed_entries: list[dict[str, Any]] = []
for collection_relative_path in sorted(grouped.keys()):
collection_entries = grouped[collection_relative_path]
collection_name = pathlib.PurePosixPath(collection_relative_path).name
collection_id = collection_key(collection_relative_path, collection_name)
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/meta.json"), None)
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/intrinsics.json"), None)
video_meta_entry = next(
(
entry
for entry in collection_entries
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
),
None,
)
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
meta = meta_payload if isinstance(meta_payload, dict) else {}
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
videos = [
entry
for entry in collection_entries
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
]
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
required = ["meta.json", "video_meta.json"]
missing_required = []
if meta_entry is None:
missing_required.append("meta.json")
if video_meta_entry is None:
missing_required.append("video_meta.json")
collection_path = source_path_join(source_path, collection_relative_path)
collection_summaries.append(
{
"id": collection_id,
"name": collection_name,
"path": collection_path,
"relativePath": collection_relative_path,
"meta": meta,
"hasIntrinsics": intrinsics_entry is not None,
"hasVideoMeta": bool(video_meta_rows),
"requiredFiles": required,
"missingRequiredFiles": missing_required,
"videoCount": len(videos),
"fileCount": len(collection_entries),
"totalBytes": total_bytes,
"valid": len(missing_required) == 0 and len(videos) > 0,
}
)
for collection_entry in collection_entries:
relative_path = str(collection_entry.get("path", "")).strip()
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
indexed_entry = {
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
"mimeType": entry_mime_type,
"source": manifest_source_ref(collection_entry, relative_path, source_path),
}
if "data" in collection_entry:
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
indexed_entry.update(
{
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
}
)
indexed_entries.append(indexed_entry)
for video_entry in videos:
relative_path = str(video_entry.get("path", "")).strip()
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
items.append(
{
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
"source": manifest_source_ref(video_entry, relative_path, source_path),
}
)
return {
"datasetType": "x-platform",
"rootPath": source_path,
"collectionCount": len(collection_summaries),
"videoCount": len(items),
"collections": collection_summaries,
"items": items,
"entries": indexed_entries,
}
def scan_xplatform_manifest_entries(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
normalized_entries: list[dict[str, Any]] = []
meta_roots: set[str] = set()
auxiliary_roots: set[str] = set()
has_root_metadata = False
for entry in entries:
relative_path = str(entry.get("path", "")).strip().lstrip("/")
if not relative_path:
continue
normalized_entry = {**entry, "path": relative_path}
normalized_entries.append(normalized_entry)
path_obj = pathlib.PurePosixPath(relative_path)
parent = path_obj.parent.as_posix()
if parent in {".", ""}:
parent = ""
file_name = path_obj.name.lower()
if file_name == "meta.json":
if parent:
meta_roots.add(parent)
else:
has_root_metadata = True
elif is_xplatform_video_meta_name(file_name) or file_name == "intrinsics.json":
if parent:
auxiliary_roots.add(parent)
else:
has_root_metadata = True
root_candidates = meta_roots or auxiliary_roots
if has_root_metadata:
root_candidates = set(root_candidates)
root_candidates.add("")
if not root_candidates:
return fallback_manifest_collections(normalized_entries, source_path)
ordered_roots = sorted(root_candidates, key=lambda value: (-len(pathlib.PurePosixPath(value).parts), value))
grouped: dict[str, list[dict[str, Any]]] = {root: [] for root in ordered_roots}
unmatched_entries: list[dict[str, Any]] = []
for entry in normalized_entries:
relative_path = str(entry.get("path", "")).strip()
matched_root = next(
(
root
for root in ordered_roots
if (
(not root)
or relative_path == root
or relative_path.startswith(root + "/")
)
),
None,
)
if matched_root is not None:
grouped.setdefault(matched_root, []).append(entry)
else:
unmatched_entries.append(entry)
collection_summaries: list[dict[str, Any]] = []
items: list[dict[str, Any]] = []
indexed_entries: list[dict[str, Any]] = []
for collection_relative_path in sorted(grouped.keys()):
collection_entries = grouped[collection_relative_path]
if not collection_entries:
continue
collection_name = (
pathlib.PurePosixPath(collection_relative_path).name
if collection_relative_path
else collection_name_from_source_path(source_path)
)
collection_id = collection_key(collection_relative_path, collection_name)
meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json"
intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json"
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None)
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None)
video_meta_entry = next(
(
entry
for entry in collection_entries
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
),
None,
)
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
meta = meta_payload if isinstance(meta_payload, dict) else {}
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
videos = [
entry
for entry in collection_entries
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
]
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
required = ["meta.json", "video_meta.json"]
missing_required = []
if meta_entry is None:
missing_required.append("meta.json")
if video_meta_entry is None:
missing_required.append("video_meta.json")
collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path)
collection_summaries.append(
{
"id": collection_id,
"name": collection_name,
"path": collection_path,
"relativePath": collection_relative_path,
"meta": meta,
"hasIntrinsics": intrinsics_entry is not None,
"hasVideoMeta": bool(video_meta_rows),
"requiredFiles": required,
"missingRequiredFiles": missing_required,
"videoCount": len(videos),
"fileCount": len(collection_entries),
"totalBytes": total_bytes,
"valid": len(missing_required) == 0 and len(videos) > 0,
}
)
for collection_entry in collection_entries:
relative_path = str(collection_entry.get("path", "")).strip()
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
indexed_entry = {
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
"mimeType": entry_mime_type,
"source": manifest_source_ref(collection_entry, relative_path, source_path),
}
if "data" in collection_entry:
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
indexed_entry.update(
{
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
}
)
indexed_entries.append(indexed_entry)
for video_entry in videos:
relative_path = str(video_entry.get("path", "")).strip()
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
items.append(
{
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
"source": manifest_source_ref(video_entry, relative_path, source_path),
}
)
if unmatched_entries:
fallback_payload = fallback_manifest_collections(unmatched_entries, source_path)
collection_summaries.extend(fallback_payload.get("collections", []))
items.extend(fallback_payload.get("items", []))
indexed_entries.extend(fallback_payload.get("entries", []))
return {
"datasetType": "x-platform",
"rootPath": source_path,
"collectionCount": len(collection_summaries),
"videoCount": len(items),
"collections": collection_summaries,
"items": items,
"entries": indexed_entries,
}
def with_xplatform_source_context(payload: dict[str, Any], source_manifest: dict[str, Any]) -> dict[str, Any]:
for key in (
"dataSource",
"sourcePath",
"scanMode",
"listedCount",
"newCount",
"sourceFingerprint",
"bucket",
"prefix",
"pattern",
"endpoint",
):
if key in source_manifest:
payload[key] = source_manifest.get(key)
return payload
def attach_xplatform_catalog_payload(payload: dict[str, Any], catalog_payload: dict[str, Any]) -> dict[str, Any]:
payload["catalogCollectionCount"] = int(catalog_payload.get("collectionCount", 0) or 0)
payload["catalogVideoCount"] = int(catalog_payload.get("videoCount", 0) or 0)
payload["catalogCollections"] = catalog_payload.get("collections", [])
payload["catalogItems"] = catalog_payload.get("items", [])
payload["catalogEntries"] = catalog_payload.get("entries", [])
return payload
def apply_source_refs_to_items(
items: list[dict[str, Any]],
source_refs: dict[str, dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]:
if not source_refs:
return items, {}
archive_sources: dict[str, dict[str, Any]] = {}
for item in items:
relative_path = str(item.get("relativePath", "")).strip()
if not relative_path:
continue
source_ref = source_refs.get(relative_path)
if source_ref:
archive_key = str(source_ref.get("archiveKey", "")).strip()
archive_preview_url = str(source_ref.get("archivePreviewURL", "")).strip()
if archive_key and archive_preview_url and archive_key not in archive_sources:
archive_sources[archive_key] = clean_metadata(
{
"name": str(source_ref.get("archiveName", "")).strip() or "archive",
"previewURL": archive_preview_url,
}
) or {}
item_source = {**source_ref, "path": item.get("path", "")}
item_source.pop("archivePreviewURL", None)
item_source.pop("archiveName", None)
item["source"] = item_source
return items, archive_sources
def scan_xplatform_dataset(root: pathlib.Path, source_refs: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]:
collections = detect_xplatform_collections(root)
collection_summaries: list[dict[str, Any]] = []
items: list[dict[str, Any]] = []
entries: list[dict[str, Any]] = []
for collection_dir in collections:
meta = optional_json_file(collection_dir / "meta.json")
meta = meta if isinstance(meta, dict) else {}
video_meta_rows, _ = load_collection_video_meta(collection_dir)
summary, collection_items, collection_entries = collection_summary(root, collection_dir, meta, video_meta_rows)
collection_summaries.append(summary)
items.extend(collection_items)
entries.extend(collection_entries)
items, archive_sources = apply_source_refs_to_items(items, source_refs or {})
entries, entry_archive_sources = apply_source_refs_to_items(entries, source_refs or {})
archive_sources.update(entry_archive_sources)
payload = {
"datasetType": "x-platform",
"rootPath": str(root),
"collectionCount": len(collection_summaries),
"videoCount": len(items),
"collections": collection_summaries,
"items": items,
"entries": entries,
}
if archive_sources:
payload["archiveSources"] = archive_sources
return payload
def load_xplatform_index_payload(
params: dict[str, str],
inputs: list[dict[str, Any]],
output_dir: pathlib.Path,
) -> dict[str, Any]:
source_manifest = first_payload_with_keys(inputs, "entries")
if source_manifest is not None:
entries = source_manifest.get("entries")
if isinstance(entries, list):
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
payload = scan_xplatform_manifest_entries(entries, manifest_source_path)
catalog_entries = source_manifest.get("catalogEntries")
if isinstance(catalog_entries, list):
catalog_payload = scan_xplatform_manifest_entries(catalog_entries, manifest_source_path)
attach_xplatform_catalog_payload(payload, catalog_payload)
return with_xplatform_source_context(payload, source_manifest)
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
if manifest_source_path and pathlib.Path(manifest_source_path).exists():
root = pathlib.Path(manifest_source_path)
return scan_xplatform_dataset(root if root.is_dir() else root.parent)
for artifact in inputs:
name = str(artifact.get("name", "")).strip().lower()
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
if name != "source-manifest.json" and not any(key in data for key in ("entriesCount", "catalogEntriesCount", "sourcePath")):
continue
raise RuntimeError("X 平台数据集索引未能读取 source-manifest 的 entries,拒绝回退到空目录扫描")
source_path = resolve_source_path(params, inputs)
if source_path is not None:
root = source_path if source_path.is_dir() else source_path.parent
return scan_xplatform_dataset(root)
materialized_root, source_refs = materialize_input_tree(inputs, output_dir)
if materialized_root is not None:
return scan_xplatform_dataset(materialized_root, source_refs)
raise RuntimeError("X 平台数据集索引需要 SOURCE_PATH 或上游输入提供真实数据目录")
def normalize_date_value(value: Any) -> str:
text = str(value or "").strip()
digits = "".join(character for character in text if character.isdigit())
if len(digits) >= 8:
return digits[:8]
return ""
def date_in_range(value: Any, start: str, end: str) -> bool:
normalized = normalize_date_value(value)
if not normalized:
return False
if start and normalized < start:
return False
if end and normalized > end:
return False
return True
def materialize_archive_member(
source: dict[str, Any],
archive_sources: dict[str, Any],
default_name: str,
output_dir: pathlib.Path,
) -> pathlib.Path | None:
archive_key = str(source.get("archiveKey", "")).strip()
archive_source = archive_sources.get(archive_key) if isinstance(archive_sources, dict) else {}
archive_source = archive_source if isinstance(archive_source, dict) else {}
preview_url = str(archive_source.get("previewURL", "")).strip()
member_path = str(source.get("archiveMemberPath", "")).strip()
decoded = decode_data_url(preview_url) if preview_url and member_path else None
if decoded is None:
return None
_, payload = decoded
archive_name = pathlib.PurePosixPath(str(archive_source.get("name", "")).strip() or "archive").name
archive_path = output_dir / "_materialized_sources" / archive_name
archive_path.parent.mkdir(parents=True, exist_ok=True)
archive_path.write_bytes(payload)
destination = output_dir / "_materialized_sources" / default_name
destination.parent.mkdir(parents=True, exist_ok=True)
if archive_name.lower().endswith(".zip"):
with zipfile.ZipFile(archive_path) as archive:
with archive.open(member_path) as handle:
destination.write_bytes(handle.read())
return destination
with tarfile.open(archive_path, "r:*") as archive:
extracted = archive.extractfile(member_path)
if extracted is None:
return None
with extracted:
destination.write_bytes(extracted.read())
return destination
def source_path_from_item(
item: dict[str, Any],
default_name: str,
output_dir: pathlib.Path,
archive_sources: dict[str, Any] | None = None,
) -> pathlib.Path:
source = item.get("source") if isinstance(item.get("source"), dict) else {}
direct = str(source.get("path", "") or item.get("path", "")).strip()
if direct and pathlib.Path(direct).exists():
return pathlib.Path(direct)
if direct.startswith("bos://"):
cached_target = workspace_cached_bos_source_path(direct, output_dir)
if cached_target is not None and cached_target.exists() and cached_target.stat().st_size > 0:
return cached_target
cached_bos_source = download_bos_source(direct, default_name, output_dir)
if cached_bos_source is not None:
return cached_bos_source
preview_url = str(source.get("previewURL", "")).strip()
if cached_target is not None and preview_url:
cached_remote_source = download_remote_source_to_path(preview_url, cached_target)
if cached_remote_source is not None:
return cached_remote_source
archive_member = materialize_archive_member(source, archive_sources or {}, default_name, output_dir)
if archive_member is not None:
return archive_member
preview_url = str(source.get("previewURL", "")).strip()
remote_path = download_remote_source(preview_url, default_name, output_dir) if preview_url else None
if remote_path is not None:
return remote_path
decoded = decode_data_url(preview_url) if preview_url else None
if decoded is None:
raise RuntimeError(f"无法获取真实输入文件: {default_name}")
_, payload = decoded
path = output_dir / "_materialized_sources" / default_name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(payload)
return path
def clip_segment_bounds(item: dict[str, Any]) -> tuple[float, float]:
start_second = float(item.get("startSecond") or 0.0)
end_second = float(item.get("endSecond") or 0.0)
duration = float(item.get("duration") or 0.0)
if end_second <= start_second and duration > 0:
end_second = start_second + duration
if end_second <= start_second:
end_second = start_second + 0.1
return start_second, end_second
def materialize_clip_from_source(
item: dict[str, Any],
output_dir: pathlib.Path,
archive_sources: dict[str, Any] | None = None,
) -> pathlib.Path | None:
source = item.get("source") if isinstance(item.get("source"), dict) else {}
source_name = (
str(item.get("sourceVideo", "")).strip()
or pathlib.PurePosixPath(str(source.get("path", "")).strip()).name
or "source.mp4"
)
clip_name = str(item.get("clipName", "")).strip() or f"{pathlib.Path(source_name).stem}_clip.mp4"
destination = output_dir / "_materialized_clips" / clip_name
if destination.exists():
return destination
start_second, end_second = clip_segment_bounds(item)
remote_source_url = remote_video_url_for_source(source) if source else ""
if remote_source_url:
try:
write_video_segment_from_remote_url(remote_source_url, destination, start_second, end_second)
return destination
except RuntimeError as exc:
log(f"[operator-runner][warn] fallback to temporary source materialization for {clip_name}: {exc}")
if not source:
return None
source_path = source_path_from_item({"source": source}, source_name, output_dir, archive_sources)
write_video_segment(source_path, destination, start_second, end_second)
return destination
def analyze_video_quality_metrics(path: pathlib.Path, clarity_scale: float) -> dict[str, Any]:
cv2, np = require_cv2()
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频文件: {path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
positions = sampled_frame_positions(frame_count)
if not positions:
positions = [0]
clarities: list[float] = []
shifts: list[tuple[float, float]] = []
skin_ratios: list[float] = []
previous_gray = None
previous_position: int | None = None
analysis_diagonal = math.sqrt((width or 1) ** 2 + (height or 1) ** 2)
for position in positions:
if frame_count > 0:
capture.set(cv2.CAP_PROP_POS_FRAMES, position)
ok, frame = capture.read()
if not ok:
continue
analysis_frame = resize_quality_frame(frame)
analysis_height, analysis_width = analysis_frame.shape[:2]
analysis_diagonal = math.sqrt((analysis_width or 1) ** 2 + (analysis_height or 1) ** 2)
gray = cv2.cvtColor(analysis_frame, cv2.COLOR_BGR2GRAY)
laplacian = cv2.Laplacian(gray, cv2.CV_64F).var()
clarities.append(float(laplacian))
if previous_gray is not None:
gap = max(position - previous_position, 1) if previous_position is not None else 1
shift, _ = cv2.phaseCorrelate(np.float32(previous_gray), np.float32(gray))
shifts.append((float(shift[0]) / gap, float(shift[1]) / gap))
mask = build_skin_mask(analysis_frame)
skin_ratios.append(largest_skin_component_ratio(mask))
previous_gray = gray
previous_position = position
clarity_raw = sum(clarities) / max(len(clarities), 1)
clarity = 1.0 - math.exp(-max(clarity_raw, 0.0) / max(clarity_scale, 1.0))
shake = motion_jitter_ratio(shifts, analysis_diagonal) if shifts else 0.0
hand_occlusion = sum(skin_ratios) / max(len(skin_ratios), 1) if skin_ratios else 0.0
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
quality_score = max(0.0, min(1.0, (clarity + (1.0 - min(shake, 1.0)) + (1.0 - min(hand_occlusion, 1.0))) / 3.0))
return {
"codec": "opencv",
"resolution": f"{width}x{height}" if width > 0 and height > 0 else "",
"frameRate": round(fps, 3) if fps > 0 else None,
"duration": round(duration, 3) if duration > 0 else None,
"clarity": round(clarity, 4),
"shake": round(shake, 4),
"handOcclusion": round(hand_occlusion, 4),
"qualityScore": round(quality_score, 4),
"sampleCount": len(clarities),
}
finally:
capture.release()
def stable_sample_hit(value: str, percent: float) -> bool:
if percent >= 100:
return True
if percent <= 0:
return False
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
score = int(digest[:8], 16) / 0xFFFFFFFF
return score <= (percent / 100.0)
def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None:
suffix = path.suffix.lower()
if suffix in TABULAR_FILE_SUFFIXES:
return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip("."))
if suffix == ".jsonl":
return preview_jsonl_payload(path)
if suffix in TEXT_FILE_SUFFIXES:
format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".")
return preview_text_payload(path, format_name)
if suffix in IMAGE_FILE_SUFFIXES:
return preview_image_payload(path, mime_type)
if suffix in ROSBAG_FILE_SUFFIXES:
return summarize_rosbag_file(path)
if suffix == ".pdf":
return {"preview": {"kind": "document", "format": "pdf"}}
if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES:
return preview_media_payload(path, mime_type or "video/mp4", "video")
if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
return preview_media_payload(path, mime_type or "audio/mpeg", "audio")
size_bytes = path.stat().st_size if path.exists() else 0
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
return {"preview": {"kind": "binary", "format": suffix.lstrip(".") or "file", "url": data_url_for_path(path, mime_type)}}
return None
def build_artifact(
run_id: str,
node_id: str,
path: pathlib.Path,
artifact_type: str,
data: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
mime_type, _ = mimetypes.guess_type(path.name)
size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8"))
sha256_value = ""
sha256_metadata: dict[str, Any] = {}
if path.exists():
sha256_value, sha256_metadata = artifact_sha256(path, size_bytes)
artifact_metadata = clean_metadata(
{
**(metadata or {}),
**sha256_metadata,
"sha256": sha256_value,
"mimeType": mime_type or "",
}
)
artifact = {
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}",
"runId": run_id,
"nodeId": node_id,
"name": path.name,
"type": artifact_type,
"size": format_size(size_bytes),
"path": str(path),
}
if output:
if str(output.get("id", "")).strip():
artifact["portId"] = str(output.get("id", "")).strip()
if str(output.get("name", "")).strip():
artifact["portName"] = str(output.get("name", "")).strip()
if artifact_metadata:
artifact["metadata"] = artifact_metadata
if data:
artifact["data"] = data
return artifact
def build_virtual_artifact(
run_id: str,
node_id: str,
name: str,
path: str,
artifact_type: str,
size_bytes: int,
data: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
artifact = {
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else name)}-{artifact_slug(name)}",
"runId": run_id,
"nodeId": node_id,
"name": name,
"type": artifact_type,
"size": format_size(max(0, size_bytes)),
"path": path,
}
if output:
if str(output.get("id", "")).strip():
artifact["portId"] = str(output.get("id", "")).strip()
if str(output.get("name", "")).strip():
artifact["portName"] = str(output.get("name", "")).strip()
if metadata:
cleaned = clean_metadata(metadata)
if cleaned:
artifact["metadata"] = cleaned
if data:
artifact["data"] = data
return artifact
def dataset_partition_keys(payload: dict[str, Any]) -> list[str]:
keys: list[str] = []
for key in DATASET_PARTITION_KEYS:
if isinstance(payload.get(key), list):
keys.append(key)
return keys
def should_externalize_structured_payload(payload: dict[str, Any]) -> bool:
if dataset_partition_keys(payload):
return True
dataset_type = str(payload.get("datasetType", "")).strip()
if dataset_type:
return True
return False
def dataset_payload_summary(payload: dict[str, Any]) -> dict[str, Any]:
summary: dict[str, Any] = {}
for key in (
"datasetType",
"count",
"itemCount",
"collectionCount",
"entryCount",
"videoCount",
"clipCount",
"annotationCount",
"packageCount",
"sampledCount",
"passedCount",
"failedCount",
"validCollectionCount",
"invalidCollectionCount",
"dataSource",
"sourcePath",
"bucket",
"prefix",
"endpoint",
"pattern",
"scanMode",
"sourceFingerprint",
"listedCount",
"newCount",
"rootPath",
"segmentDurationSeconds",
):
if key in payload:
summary[key] = json_compatible(payload[key])
if "preview" in payload:
summary["preview"] = json_compatible(payload.get("preview"))
for key in dataset_partition_keys(payload):
value = payload.get(key)
if isinstance(value, list):
summary[f"{key}Count"] = len(value)
if "preview" not in summary:
summary["preview"] = {
"kind": "dataset",
"format": str(payload.get("datasetType", "")).strip() or "json",
"counts": {key: summary[key] for key in summary.keys() if key.endswith("Count") or key == "count"},
}
return summary
def chunk_list(values: list[Any], chunk_size: int = DATASET_PARTITION_SIZE) -> list[list[Any]]:
if chunk_size <= 0:
chunk_size = DATASET_PARTITION_SIZE
return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] or [[]]
def externalize_structured_payload(path: pathlib.Path, payload: dict[str, Any]) -> dict[str, Any]:
normalized_payload = json_compatible(payload)
dataset_dir = path.parent / "_datasets" / artifact_slug(path.stem or path.name)
partitions_dir = dataset_dir / "partitions"
partitions_dir.mkdir(parents=True, exist_ok=True)
partition_files: dict[str, list[str]] = {}
partition_count = 0
partition_keys = dataset_partition_keys(normalized_payload)
for key in partition_keys:
values = normalized_payload.get(key)
if not isinstance(values, list):
continue
files: list[str] = []
for index, chunk in enumerate(chunk_list(values), start=1):
partition_path = partitions_dir / f"{key}-p{index:06d}.json"
write_json_file(partition_path, chunk)
files.append(str(partition_path))
partition_files[key] = files
partition_count += len(files)
manifest = {
"kind": "dataset",
"datasetType": str(normalized_payload.get("datasetType", "")).strip() or "json",
"payloadPath": str(path),
"partitionDir": str(partitions_dir),
"partitionKeys": partition_keys,
"partitionFiles": partition_files,
"summary": dataset_payload_summary(normalized_payload),
"generatedAt": now_iso(),
}
manifest_path = write_json_file(dataset_dir / "manifest.json", manifest)
data_ref = {
"kind": "dataset",
"datasetType": manifest["datasetType"],
"manifestPath": str(manifest_path),
"payloadPath": str(path),
"partitionDir": str(partitions_dir),
"partitionCount": partition_count,
}
item_count = 0
for key in partition_keys:
values = normalized_payload.get(key)
if isinstance(values, list):
item_count += len(values)
if item_count > 0:
data_ref["itemCount"] = item_count
summary = dataset_payload_summary(normalized_payload)
summary["dataRef"] = data_ref
return summary
def write_json_artifact(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
filename: str,
payload: dict[str, Any],
artifact_type: str = "json",
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
normalized_payload = json_compatible(payload)
path = output_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
json.dump(normalized_payload, handle, ensure_ascii=False, indent=2)
artifact_data: dict[str, Any] | None = normalized_payload
if should_externalize_structured_payload(normalized_payload):
artifact_data = externalize_structured_payload(path, normalized_payload)
return build_artifact(run_id, node_id, path, artifact_type, data=artifact_data, output=output)
def summarize_json_like(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return {"kind": "object", "keys": sorted(value.keys())}
if isinstance(value, list):
return {"kind": "list", "length": len(value)}
return {"kind": type(value).__name__, "value": value}
def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]:
data = artifact.get("data")
if isinstance(data, dict):
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
payload_path = str(data_ref.get("payloadPath", "")).strip()
if payload_path:
payload = load_payload_from_file(pathlib.Path(payload_path))
if isinstance(payload, dict):
return payload
if isinstance(data, dict):
return data
path_value = artifact.get("path")
if isinstance(path_value, str) and path_value:
payload = load_payload_from_file(pathlib.Path(path_value))
if payload is not None:
return payload
summary = {
"name": artifact.get("name", ""),
"type": artifact.get("type", ""),
"path": artifact.get("path", ""),
"size": artifact.get("size", ""),
}
return {key: value for key, value in summary.items() if value}
def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]:
merged: dict[str, Any] = {}
for value in values:
merged.update(value)
return merged
def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]:
entries: list[dict[str, Any]] = []
if not base_path.exists():
return entries
if base_path.is_file():
relative = base_path.name
if pattern and not fnmatch.fnmatch(relative, pattern):
return entries
return [
{
"path": relative,
"sizeBytes": base_path.stat().st_size,
"sha256": sha256_file(base_path),
}
]
for candidate in sorted(base_path.rglob("*")):
if candidate.is_dir():
continue
relative = candidate.relative_to(base_path).as_posix()
if pattern and not fnmatch.fnmatch(relative, pattern):
continue
entries.append(
{
"path": relative,
"sizeBytes": candidate.stat().st_size,
"sha256": sha256_file(candidate),
}
)
return entries
def list_bos_entries(
bucket: str,
prefix: str,
pattern: str,
endpoint: str,
access_key_id: str,
secret_access_key: str,
) -> list[dict[str, Any]]:
client, scheme = build_bos_client(endpoint, access_key_id, secret_access_key)
normalized_prefix = bos_prefix(prefix)
prefix_with_slash = f"{normalized_prefix}/" if normalized_prefix else ""
entries: list[dict[str, Any]] = []
for candidate in client.list_all_objects(bucket, prefix=normalized_prefix or None):
key = text_value(getattr(candidate, "key", "")).strip().lstrip("/")
if not key or key.endswith("/"):
continue
if normalized_prefix:
if key == normalized_prefix:
continue
if not key.startswith(prefix_with_slash):
continue
relative_path = key[len(prefix_with_slash):]
else:
relative_path = key
if not relative_path:
continue
if pattern and not fnmatch.fnmatch(relative_path, pattern):
continue
size_bytes = int(getattr(candidate, "size", 0) or 0)
mime_type = mimetypes.guess_type(relative_path)[0] or ""
storage_path = canonical_bos_path(bucket, key)
entry: dict[str, Any] = {
"path": relative_path,
"storagePath": storage_path,
"sizeBytes": size_bytes,
"mimeType": mime_type,
"etag": text_value(getattr(candidate, "e_tag", "") or getattr(candidate, "etag", "")),
"versionId": text_value(getattr(candidate, "version_id", "") or getattr(candidate, "versionId", "")),
"lastModified": text_value(getattr(candidate, "last_modified", "") or getattr(candidate, "lastModified", "")),
"source": {"path": storage_path},
}
file_name = pathlib.PurePosixPath(relative_path).name.lower()
should_inline_json = relative_path.lower().endswith(".json") and (
file_name in XPLATFORM_INLINE_JSON_FILES or size_bytes <= PREVIEW_INLINE_BINARY_LIMIT
)
if should_inline_json:
raw = client.get_object_as_string(bucket, key)
payload_text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
try:
entry["data"] = json.loads(payload_text)
except Exception:
entry["data"] = {"value": payload_text}
elif pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
entry["source"]["previewURL"] = client.generate_pre_signed_url(
bucket,
key,
expiration_in_seconds=3600,
protocol=scheme,
)
entries.append(clean_metadata(entry) or {})
return entries
def source_entry_fingerprint(data_source: str, bucket: str, entry: dict[str, Any]) -> str:
normalized_source = (data_source or "BOS").strip().upper() or "BOS"
relative_path = str(entry.get("path", "")).strip()
version_id = str(entry.get("versionId", "")).strip()
sha256 = str(entry.get("sha256", "")).strip()
etag = str(entry.get("etag", "")).strip()
size_bytes = str(entry.get("sizeBytes", "")).strip()
last_modified = str(entry.get("lastModified", "")).strip()
parts = [normalized_source, str(bucket or "").strip(), relative_path]
if version_id:
parts.append(f"version:{version_id}")
elif sha256:
parts.append(f"sha256:{sha256}")
else:
parts.extend([f"etag:{etag}", f"size:{size_bytes}", f"lastModified:{last_modified}"])
raw = "\n".join(parts).encode("utf-8")
return hashlib.sha256(raw).hexdigest()
def incremental_unit(params: dict[str, str]) -> str:
return "DIRECTORY" if (params.get("INCREMENTAL_UNIT") or "").strip().upper() == "DIRECTORY" else "OBJECT"
def incremental_directory_depth(params: dict[str, str]) -> int:
raw = (params.get("INCREMENTAL_DIRECTORY_DEPTH") or "").strip()
try:
value = int(raw)
except Exception:
value = 1
return value if value > 0 else 1
def directory_incremental_key(relative_path: str, depth: int) -> str:
clean_relative = str(relative_path or "").strip().lstrip("/")
if not clean_relative:
return "__root__"
parts = [segment for segment in pathlib.PurePosixPath(clean_relative).parts if segment not in ("", ".")]
if len(parts) <= 1:
return f"FILE::{clean_relative}"
directory_parts = parts[:-1]
take = min(max(depth, 1), len(directory_parts))
return "/".join(directory_parts[:take])
def incremental_source_state(params: dict[str, str]) -> dict[str, Any]:
raw = params.get("SOURCE_STATE_JSON", "").strip()
if not raw:
return {"seenFingerprints": []}
try:
parsed = json.loads(raw)
except Exception:
return {"seenFingerprints": []}
if not isinstance(parsed, dict):
return {"seenFingerprints": []}
seen = parsed.get("seenFingerprints")
if not isinstance(seen, list):
parsed["seenFingerprints"] = []
else:
parsed["seenFingerprints"] = [str(item).strip() for item in seen if str(item).strip()]
parsed["incrementalUnit"] = "DIRECTORY" if str(parsed.get("incrementalUnit", "")).strip().upper() == "DIRECTORY" else "OBJECT"
try:
parsed["directoryDepth"] = int(parsed.get("directoryDepth", 1) or 1)
except Exception:
parsed["directoryDepth"] = 1
if parsed["directoryDepth"] <= 0:
parsed["directoryDepth"] = 1
return parsed
def apply_incremental_source_filter(
entries: list[dict[str, Any]],
data_source: str,
bucket: str,
params: dict[str, str],
source_path: str,
prefix: str,
pattern: str,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
state = incremental_source_state(params)
seen = {str(item).strip() for item in state.get("seenFingerprints", []) if str(item).strip()}
scan_started_at = now_iso()
mode = incremental_unit(params)
depth = incremental_directory_depth(params)
emitted: list[dict[str, Any]] = []
updated_seen = set(seen)
if mode == "DIRECTORY":
grouped_entries: dict[str, list[dict[str, Any]]] = {}
for entry in entries:
group_key = directory_incremental_key(str(entry.get("path", "")).strip(), depth)
grouped_entries.setdefault(group_key, []).append(entry)
for group_key in sorted(grouped_entries.keys()):
group_entries = grouped_entries[group_key]
if group_key in updated_seen:
continue
updated_seen.add(group_key)
for entry in group_entries:
enriched = dict(entry)
enriched["fingerprint"] = source_entry_fingerprint(data_source, bucket, entry)
enriched["incrementalKey"] = group_key
emitted.append(enriched)
else:
for entry in entries:
fingerprint = source_entry_fingerprint(data_source, bucket, entry)
enriched = dict(entry)
enriched["fingerprint"] = fingerprint
enriched["incrementalKey"] = fingerprint
if fingerprint in updated_seen:
continue
updated_seen.add(fingerprint)
emitted.append(enriched)
scan_completed_at = now_iso()
checkpoint = {
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
"scanMode": "INCREMENTAL",
"incrementalUnit": mode,
"directoryDepth": depth,
"dataSource": data_source,
"sourcePath": source_path,
"bucket": bucket,
"prefix": prefix,
"pattern": pattern,
"seenFingerprints": sorted(updated_seen),
"listedCount": len(entries),
"newCount": len(emitted),
"scanStartedAt": scan_started_at,
"scanCompletedAt": scan_completed_at,
}
return emitted, checkpoint
def preview_payload_for_remote_entry(relative_path: str, mime_type: str, preview_url: str) -> dict[str, Any] | None:
normalized_preview_url = text_value(preview_url).strip()
if not normalized_preview_url:
return None
suffix = pathlib.PurePosixPath(relative_path).suffix.lower()
kind = "binary"
if suffix in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
kind = "video"
elif suffix in IMAGE_FILE_SUFFIXES or mime_type.startswith("image/"):
kind = "image"
elif suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
kind = "audio"
return {"preview": {"kind": kind, "format": suffix.lstrip(".") or kind, "url": normalized_preview_url}}
def source_entry_artifact(
run_id: str,
node_id: str,
entry: dict[str, Any],
source_path: str,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
relative_path = str(entry.get("path", "")).strip().lstrip("/")
source = manifest_source_ref(entry, relative_path, source_path)
storage_path = str(source.get("path", "")).strip() or source_path_join(source_path, relative_path)
mime_type = str(entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
artifact_type = infer_port_type(pathlib.Path(relative_path))
payload: dict[str, Any] | None = None
if "data" in entry:
payload = normalize_json_payload(entry.get("data"))
else:
preview_payload = preview_payload_for_remote_entry(relative_path, mime_type, str(source.get("previewURL", "")).strip())
payload = preview_payload if preview_payload else None
if payload is not None:
payload["relativePath"] = relative_path
if source:
payload["source"] = source
metadata = clean_metadata({"mimeType": mime_type, "sha256": entry.get("sha256", "")})
return build_virtual_artifact(
run_id,
node_id,
pathlib.PurePosixPath(relative_path).name or "artifact",
storage_path,
artifact_type,
int(entry.get("sizeBytes", 0) or 0),
data=payload,
metadata=metadata,
output=output,
)
def collect_output_artifacts(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
artifacts: list[dict[str, Any]] = []
for path in sorted(output_dir.rglob("*")):
if path.is_dir():
continue
artifact_type = infer_port_type(path)
mime_type, _ = mimetypes.guess_type(path.name)
if artifact_type == "json":
payload = load_payload_from_file(path)
else:
payload = extract_preview_data(path, artifact_type, mime_type or "")
binding = output
if binding is None and node_data is not None:
binding = output_binding(node_data)
artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding))
return artifacts
def hyperparams(node_data: dict[str, Any]) -> dict[str, str]:
values: dict[str, str] = {}
for item in node_data.get("hyperparams", []) or []:
key = str(item.get("key", "")).strip()
if not key:
continue
values[key] = str(item.get("value", ""))
return values
def parse_json_param(params: dict[str, str], key: str, default: Any) -> Any:
raw = params.get(key, "").strip()
if not raw:
return default
try:
return json.loads(raw)
except Exception:
return default
def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [load_artifact_payload(item) for item in inputs]
def collect_items(payloads: list[dict[str, Any]]) -> list[Any]:
items: list[Any] = []
for payload in payloads:
if isinstance(payload.get("items"), list):
items.extend(payload["items"])
elif isinstance(payload.get("entries"), list):
items.extend(payload["entries"])
elif isinstance(payload.get("rows"), list):
items.extend(payload["rows"])
else:
items.append(payload)
return items
def normalize_mongo_document(item: Any) -> dict[str, Any]:
if isinstance(item, dict):
if set(item.keys()) == {"value"} and isinstance(item.get("value"), dict):
return json_compatible(item["value"])
return json_compatible(item)
return {"value": json_compatible(item)}
def documents_from_inputs(payloads: list[dict[str, Any]], document_path: str) -> list[dict[str, Any]]:
raw_items: list[Any] = []
path = str(document_path or "").strip()
if path:
for payload in payloads:
value = get_field_value(payload, path)
if isinstance(value, list):
raw_items.extend(value)
elif value is not None:
raw_items.append(value)
else:
raw_items = collect_items(payloads)
return [normalize_mongo_document(item) for item in raw_items]
def build_mongo_client(uri: str):
try:
from pymongo import MongoClient
except Exception as exc:
raise RuntimeError("pymongo is required for MongoDB 写入算子") from exc
return MongoClient(uri, serverSelectionTimeoutMS=5000)
def get_field_value(value: Any, field_path: str) -> Any:
current = value
for token in [part for part in field_path.split(".") if part]:
if isinstance(current, dict):
current = current.get(token)
continue
if isinstance(current, list) and token.isdigit():
index = int(token)
if 0 <= index < len(current):
current = current[index]
continue
return None
return current
def compare_values(actual: Any, operator: str, expected: Any) -> bool:
if operator == "exists":
return actual is not None
if operator == "missing":
return actual is None
if operator == "contains":
if isinstance(actual, list):
return expected in actual
return str(expected) in str(actual)
if operator == "in":
if isinstance(expected, list):
return actual in expected
return str(actual) in str(expected)
if operator in {"gt", "gte", "lt", "lte"}:
try:
actual_number = float(actual)
expected_number = float(expected)
except Exception:
return False
if operator == "gt":
return actual_number > expected_number
if operator == "gte":
return actual_number >= expected_number
if operator == "lt":
return actual_number < expected_number
return actual_number <= expected_number
if operator == "ne":
return actual != expected
return actual == expected
def flatten_context(value: Any, prefix: str = "") -> dict[str, str]:
context: dict[str, str] = {}
if isinstance(value, dict):
for key, item in value.items():
next_prefix = f"{prefix}.{key}" if prefix else str(key)
context.update(flatten_context(item, next_prefix))
if prefix:
context[prefix] = json.dumps(value, ensure_ascii=False)
return context
if isinstance(value, list):
for index, item in enumerate(value[:8]):
next_prefix = f"{prefix}.{index}" if prefix else str(index)
context.update(flatten_context(item, next_prefix))
if prefix:
context[prefix] = json.dumps(value, ensure_ascii=False)
return context
if prefix:
context[prefix] = "" if value is None else str(value)
return context
def render_template_text(template_text: str, context: dict[str, str]) -> str:
rendered = template_text
for key, value in context.items():
rendered = rendered.replace("{{" + key + "}}", value)
safe_context = {key: value for key, value in context.items()}
class SafeDict(dict):
def __missing__(self, key: str) -> str:
return "{" + key + "}"
try:
rendered = rendered.format_map(SafeDict(safe_context))
except Exception:
pass
return rendered
def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for item in collect_items(payloads):
if isinstance(item, dict):
rows.append(item)
else:
rows.append({"value": item})
return rows
def write_text_artifact(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
filename: str,
content: str,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
path = output_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
mime_type, _ = mimetypes.guess_type(path.name)
preview = extract_preview_data(path, infer_port_type(path), mime_type or "")
return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output)
def write_csv_artifact(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
filename: str,
rows: list[dict[str, Any]],
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
path = output_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
columns: list[str] = []
for row in rows:
for key in row.keys():
if key not in columns:
columns.append(key)
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=columns or ["value"])
writer.writeheader()
if rows:
writer.writerows(rows)
preview = preview_tabular_payload(path, ",", "csv")
return build_artifact(run_id, node_id, path, "file", data=preview, output=output)
def write_video_segment(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
cv2, _ = require_cv2()
capture = cv2.VideoCapture(str(source_path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频文件: {source_path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
start_frame = max(0, int(start_second * fps))
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
if frame_count > 0:
end_frame = min(end_frame, frame_count)
capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
destination.parent.mkdir(parents=True, exist_ok=True)
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
if not writer.isOpened():
raise RuntimeError(f"无法创建视频切片: {destination}")
written = 0
try:
for _ in range(start_frame, end_frame):
ok, frame = capture.read()
if not ok:
break
writer.write(frame)
written += 1
finally:
writer.release()
duration = written / fps if fps > 0 else 0.0
return {
"frameRate": round(fps, 3),
"frameCount": written,
"resolution": f"{width}x{height}",
"duration": round(duration, 3),
}
finally:
capture.release()
def ffmpeg_available() -> bool:
return shutil.which("ffmpeg") is not None
def run_ffmpeg(args: list[str]) -> None:
completed = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if completed.returncode == 0:
return
stderr = (completed.stderr or "").strip()
stdout = (completed.stdout or "").strip()
message = stderr or stdout or "unknown ffmpeg error"
raise RuntimeError(f"ffmpeg execution failed: {message}")
def write_video_segment_with_ffmpeg(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
if not ffmpeg_available():
raise RuntimeError("ffmpeg is not available in operator-runner image")
duration = max(float(end_second) - float(start_second), 0.001)
destination.parent.mkdir(parents=True, exist_ok=True)
copy_args = [
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
"-ss",
f"{max(float(start_second), 0.0):.3f}",
"-i",
str(source_path),
"-t",
f"{duration:.3f}",
"-map",
"0:v:0",
"-map",
"0:a?",
"-c",
"copy",
str(destination),
]
reencode_args = [
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
"-ss",
f"{max(float(start_second), 0.0):.3f}",
"-i",
str(source_path),
"-t",
f"{duration:.3f}",
"-map",
"0:v:0",
"-map",
"0:a?",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"23",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
str(destination),
]
try:
run_ffmpeg(copy_args)
except RuntimeError:
run_ffmpeg(reencode_args)
metrics = inspect_video_file(destination)
return {
"frameRate": metrics.get("frameRate"),
"frameCount": metrics.get("frameCount"),
"resolution": metrics.get("resolution"),
"duration": metrics.get("duration", round(duration, 3)),
}
def write_video_segments(
source_path: pathlib.Path | str,
segments: list[dict[str, Any]],
) -> list[dict[str, Any]]:
cv2, _ = require_cv2()
capture = cv2.VideoCapture(str(source_path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频文件: {source_path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
normalized_segments: list[dict[str, Any]] = []
for raw in segments:
destination = pathlib.Path(raw.get("destination", ""))
start_second = float(raw.get("startSecond") or 0.0)
end_second = float(raw.get("endSecond") or 0.0)
start_frame = max(0, int(start_second * fps))
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
if frame_count > 0:
end_frame = min(end_frame, frame_count)
normalized_segments.append(
{
**raw,
"destination": destination,
"startFrame": start_frame,
"endFrame": end_frame,
}
)
normalized_segments.sort(key=lambda item: (int(item["startFrame"]), str(item["destination"])))
current_frame = 0
if normalized_segments:
initial_frame = int(normalized_segments[0]["startFrame"])
if initial_frame > 0:
capture.set(cv2.CAP_PROP_POS_FRAMES, initial_frame)
current_frame = initial_frame
results: list[dict[str, Any]] = []
for segment in normalized_segments:
start_frame = int(segment["startFrame"])
end_frame = int(segment["endFrame"])
destination = pathlib.Path(segment["destination"])
if current_frame < start_frame:
while current_frame < start_frame:
ok, _ = capture.read()
if not ok:
current_frame = end_frame
break
current_frame += 1
destination.parent.mkdir(parents=True, exist_ok=True)
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
if not writer.isOpened():
raise RuntimeError(f"无法创建视频切片: {destination}")
written = 0
try:
while current_frame < end_frame:
ok, frame = capture.read()
if not ok:
break
writer.write(frame)
written += 1
current_frame += 1
finally:
writer.release()
results.append(
{
**segment,
"frameRate": round(fps, 3),
"frameCount": written,
"resolution": f"{width}x{height}",
"duration": round(written / fps, 3) if fps > 0 else 0.0,
}
)
return results
finally:
capture.release()
def write_video_segment_from_remote_url(source_url: str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
return write_video_segment(source_url, destination, start_second, end_second)
def annotate_video_clip(path: pathlib.Path, frame_interval_seconds: float) -> dict[str, Any]:
cv2, np = require_cv2()
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频切片: {path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
step = max(1, int(round(max(frame_interval_seconds, 0.1) * fps)))
previous_gray = None
samples: list[tuple[float, float]] = []
objects: set[str] = set()
frame_index = 0
while True:
ok, frame = capture.read()
if not ok:
break
if frame_index % step != 0:
frame_index += 1
continue
timestamp = frame_index / fps if fps > 0 else 0.0
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
motion = 0.0
if previous_gray is not None:
diff = cv2.absdiff(previous_gray, gray)
motion = float(diff.mean() / 255.0)
previous_gray = gray
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
mask = cv2.inRange(ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8))
if float(mask.mean() / 255.0) > 0.1:
objects.add("手部")
_, threshold = cv2.threshold(gray, 24, 255, cv2.THRESH_BINARY)
contours, _ = cv2.findContours(threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
if any(cv2.contourArea(contour) >= 64 for contour in contours):
objects.add("前景目标")
samples.append((timestamp, motion))
frame_index += 1
if not samples:
samples.append((0.0, 0.0))
def label_for_motion(motion: float) -> tuple[str, float]:
if motion < 0.015:
return "静止观察", 0.88
if motion < 0.045:
return "平稳活动", 0.79
return "快速移动", 0.71
actions: list[dict[str, Any]] = []
last_timestamp = duration
for timestamp, motion in samples:
label, confidence = label_for_motion(motion)
last_timestamp = timestamp
if not actions:
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
continue
current = actions[-1]
if current["label"] != label:
current["end"] = round(timestamp, 3)
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
else:
current["end"] = round(timestamp, 3)
current["confidence"] = round(min(float(current["confidence"]), confidence), 3)
if actions:
actions[-1]["end"] = round(max(duration, last_timestamp), 3)
min_confidence = min(float(item["confidence"]) for item in actions) if actions else 0.0
return {
"duration": round(duration, 3),
"actions": actions,
"objects": sorted(objects),
"minConfidence": round(min_confidence, 3),
"frameIntervalSeconds": frame_interval_seconds,
}
finally:
capture.release()
def builtin_source(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT"))
data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper()
if not data_source:
data_source = "INLINE" if inline_source else "BOS"
source_path = params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "").strip()
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
scan_mode = (params.get("SCAN_MODE") or "").strip().upper()
unit = incremental_unit(params)
depth = incremental_directory_depth(params)
source_bucket, source_prefix = parse_bos_path(source_path)
if not bucket and source_bucket:
bucket = source_bucket
if not prefix and source_prefix:
prefix = source_prefix
entries: list[dict[str, Any]] = []
resolved_source_path = source_path
if params.get("SOURCE_JSON"):
parsed = json.loads(params["SOURCE_JSON"])
if isinstance(parsed, list):
entries = [{"value": item} for item in parsed]
else:
entries = [{"value": parsed}]
elif params.get("SOURCE_TEXT"):
entries = [{"text": params["SOURCE_TEXT"]}]
elif data_source == "BOS" or (bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key):
cleaned_prefix = bos_prefix(prefix)
entries = list_bos_entries(
bucket,
cleaned_prefix,
pattern,
bos_endpoint,
bos_access_key_id,
bos_secret_access_key,
)
resolved_source_path = canonical_bos_path(bucket, cleaned_prefix)
elif source_path:
entries = list_source_entries(pathlib.Path(source_path), pattern)
elif prefix and pathlib.Path(prefix).exists():
entries = list_source_entries(pathlib.Path(prefix), pattern)
elif inputs:
entries = [load_artifact_payload(item) for item in inputs]
listed_entries = list(entries)
checkpoint_payload = None
if scan_mode == "INCREMENTAL":
entries, checkpoint_payload = apply_incremental_source_filter(
listed_entries,
data_source,
bucket,
params,
resolved_source_path,
prefix,
pattern,
)
payload = {
"dataSource": data_source,
"bucket": bucket,
"prefix": prefix,
"endpoint": bos_endpoint,
"pattern": pattern,
"scanMode": scan_mode or "FULL",
"incrementalUnit": unit,
"directoryDepth": depth,
"sourcePath": resolved_source_path,
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
"listedCount": len(listed_entries),
"newCount": len(entries) if checkpoint_payload is not None else len(entries),
"entries": entries,
"catalogEntries": listed_entries,
}
artifacts = [
write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))
]
if checkpoint_payload is not None:
artifacts.append(write_json_artifact(run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload))
if data_source == "BOS":
for entry in entries:
artifacts.append(source_entry_artifact(run_id, node_id, entry, resolved_source_path, output=output_binding(node_data)))
elif source_path and pathlib.Path(source_path).exists():
source_root = pathlib.Path(source_path)
staged_root = output_dir / "_source_tree"
for entry in entries:
relative_path = str(entry.get("path", "")).strip()
if not relative_path:
continue
candidate = source_root / relative_path if source_root.is_dir() else source_root
if not candidate.exists() or not candidate.is_file():
continue
destination = staged_root / relative_path
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(candidate, destination)
artifacts.extend(collect_output_artifacts(run_id, node_id, staged_root, node_data))
return artifacts
def builtin_merge(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = payloads_from_inputs(inputs)
merged = merge_dicts(payloads)
merged["inputCount"] = len(payloads)
return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))]
def builtin_intersection(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = payloads_from_inputs(inputs)
if not payloads:
return [
write_json_artifact(
run_id,
node_id,
output_dir,
"intersection.json",
{"keys": [], "inputCount": 0},
output=output_binding(node_data),
)
]
key_sets = [set(payload.keys()) for payload in payloads]
common_keys = sorted(set.intersection(*key_sets)) if key_sets else []
payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}}
return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))]
def builtin_distribution(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = payloads_from_inputs(inputs)
items = collect_items(payloads)
outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}]
partitions: list[list[Any]] = [[] for _ in outputs]
for index, item in enumerate(items):
partitions[index % len(outputs)].append(item)
artifacts: list[dict[str, Any]] = []
for index, output in enumerate(outputs):
filename = f"{output.get('name', f'queue-{index}')}.json"
payload = {"items": partitions[index], "count": len(partitions[index])}
artifacts.append(
write_json_artifact(
run_id,
node_id,
output_dir,
filename,
payload,
str(output.get("type", "json")),
output=output,
)
)
return artifacts
def builtin_upload(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip()
bucket = str(node_data.get("bucket", "")).strip()
prefix = str(node_data.get("prefix", "")).strip()
parsed_bucket, parsed_prefix = parse_bos_path(upload_path)
if parsed_bucket:
bucket = parsed_bucket
if parsed_prefix or upload_path.endswith("/"):
prefix = parsed_prefix
data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper()
bos_endpoint = params.get("BOS_ENDPOINT", "").strip()
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip()
copied_paths: list[str] = []
bundle_inputs: list[dict[str, Any]] = []
staging_dir = output_dir / "uploaded"
staging_dir.mkdir(parents=True, exist_ok=True)
package_manifest_payload = first_payload_with_keys(inputs, "packageCount", "items")
upload_inputs = list(inputs)
if package_manifest_payload is not None:
package_names = package_related_artifact_names(package_manifest_payload)
upload_inputs = [
artifact
for artifact in inputs
if str(artifact.get("name", "")).strip() not in package_names
]
upload_inputs.extend(
materialize_package_artifacts_from_manifest(
run_id,
node_id,
staging_dir,
package_manifest_payload,
node_data,
)
)
if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key:
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
temp_paths = [
str(item.get("path", "")).strip()
for item in upload_inputs
if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip()
]
input_root = common_input_root(temp_paths)
uploaded_objects: list[dict[str, Any]] = []
remote_artifacts: list[dict[str, Any]] = []
for index, artifact in enumerate(upload_inputs):
payload = load_artifact_payload(artifact)
bundle_inputs.append(payload)
fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}"
relative_path = upload_relative_path(artifact, input_root, fallback_name)
temp_source = materialized_upload_source(artifact, staging_dir, relative_path)
if temp_source is None or not temp_source.exists():
continue
mime_type = (
str((artifact.get("metadata") or {}).get("mimeType", "")).strip()
if isinstance(artifact.get("metadata"), dict)
else ""
) or mimetypes.guess_type(temp_source.name)[0] or ""
object_key = bos_key_join(prefix, relative_path)
put_kwargs = {"content_type": mime_type} if mime_type else {}
try:
size_bytes = temp_source.stat().st_size
except OSError:
size_bytes = 0
log(
f"[operator-runner][info] upload starts: {temp_source} -> {canonical_bos_path(bucket, object_key)} "
f"({size_bytes} bytes)"
)
client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs)
remote_path = canonical_bos_path(bucket, object_key)
log(
f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} "
f"({size_bytes} bytes)"
)
preview_url = ""
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
preview_url = text_value(
client.generate_pre_signed_url(
bucket,
object_key,
expiration_in_seconds=3600,
protocol=scheme,
)
).strip()
uploaded_objects.append(
clean_metadata(
{
"name": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": remote_path,
"previewURL": preview_url,
"sizeBytes": temp_source.stat().st_size,
"mimeType": mime_type,
}
)
or {}
)
artifact_type = str(artifact.get("type", "")).strip() or infer_port_type(temp_source)
artifact_payload: dict[str, Any] | None = None
if artifact_type == "json":
loaded = load_payload_from_file(temp_source)
if loaded is not None:
artifact_payload = normalize_json_payload(loaded)
else:
artifact_payload = preview_payload_for_remote_entry(relative_path, mime_type, preview_url)
if artifact_payload is None:
artifact_payload = extract_preview_data(temp_source, artifact_type, mime_type)
if artifact_payload is not None:
artifact_payload["relativePath"] = relative_path
artifact_payload["source"] = clean_metadata(
{
"path": remote_path,
"previewURL": preview_url,
"relativePath": relative_path,
"mimeType": mime_type,
}
) or {}
source_size_bytes = temp_source.stat().st_size
source_sha256, source_sha256_metadata = artifact_sha256(temp_source, source_size_bytes)
remote_artifacts.append(
build_virtual_artifact(
run_id,
node_id,
pathlib.PurePosixPath(relative_path).name,
remote_path,
artifact_type,
source_size_bytes,
data=artifact_payload,
metadata={
**source_sha256_metadata,
"sha256": source_sha256,
"mimeType": mime_type,
},
output=output_binding(node_data),
)
)
payload = {
"copiedPaths": copied_paths,
"uploadedCount": len(uploaded_objects),
"uploadedObjects": uploaded_objects,
"items": bundle_inputs,
"inputCount": len(inputs),
"dataSource": "BOS",
"bucket": bucket,
"prefix": prefix,
"destinationPath": canonical_bos_path(bucket, prefix),
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
artifacts.extend(remote_artifacts)
return artifacts
for artifact in upload_inputs:
payload = load_artifact_payload(artifact)
bundle_inputs.append(payload)
source_path = artifact.get("path")
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
source = pathlib.Path(source_path)
destination = staging_dir / source.name
if source.resolve() != destination.resolve():
shutil.copy2(source, destination)
copied_paths.append(str(destination))
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir))
return artifacts
def builtin_json_extract(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_path = params.get("FIELD_PATH", "").strip()
payloads = payloads_from_inputs(inputs)
payload = payloads[0] if payloads else {}
extracted = get_field_value(payload, field_path) if field_path else payload
result = {"fieldPath": field_path, "value": extracted}
return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))]
def builtin_json_filter(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_path = params.get("FIELD_PATH", "").strip()
operator = params.get("OPERATOR", "eq").strip().lower() or "eq"
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
rows = collect_items(payloads_from_inputs(inputs))
filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
payload = {
"fieldPath": field_path,
"operator": operator,
"expected": expected,
"items": filtered,
"count": len(filtered),
}
return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))]
def builtin_json_map(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_map = parse_json_param(params, "FIELD_MAP_JSON", {})
rows = rows_from_payloads(payloads_from_inputs(inputs))
mapped = [{str(field_map.get(key, key)): value for key, value in row.items()} for row in rows]
payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)}
return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))]
def builtin_mongodb_write(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
mongo_target = params.get("MONGO_TARGET", "").strip().lower() or "business"
explicit_mongo_uri = params.get("MONGO_URI", "").strip() or os.getenv("MONGO_URI", "").strip()
explicit_mongo_db = params.get("MONGO_DB", "").strip() or os.getenv("MONGO_DB", "").strip()
collection_name = params.get("MONGO_COLLECTION", "").strip()
mode = params.get("MONGO_MODE", "insert_many").strip().lower() or "insert_many"
upsert_key = params.get("UPSERT_KEY", "_id").strip() or "_id"
document_path = params.get("DOCUMENT_PATH", "").strip()
mongo_uri = explicit_mongo_uri
mongo_db = explicit_mongo_db
if not mongo_uri:
if mongo_target == "platform":
mongo_uri = os.getenv("PLATFORM_MONGO_URI", "").strip()
mongo_db = mongo_db or os.getenv("PLATFORM_MONGO_DB", "").strip()
else:
mongo_uri = os.getenv("BUSINESS_MONGO_URI", "").strip()
mongo_db = mongo_db or os.getenv("BUSINESS_MONGO_DB", "").strip()
if not mongo_uri:
raise RuntimeError(f"MongoDB 写入缺少可用连接:target={mongo_target}")
if not mongo_db:
raise RuntimeError(f"MongoDB 写入缺少数据库名:target={mongo_target}")
if not collection_name:
raise RuntimeError("MONGO_COLLECTION is required for MongoDB 写入")
if mongo_target == "platform" and collection_name.lower() in {item.lower() for item in RESERVED_MONGO_COLLECTIONS}:
raise RuntimeError(f"collection {collection_name} is reserved and cannot be written by workflow operators")
if mode not in {"insert_many", "upsert"}:
raise RuntimeError("MONGO_MODE must be insert_many or upsert")
documents = documents_from_inputs(payloads_from_inputs(inputs), document_path)
if not documents:
raise RuntimeError("MongoDB 写入未收到可写入的文档")
client = build_mongo_client(mongo_uri)
try:
collection = client[mongo_db][collection_name]
summary: dict[str, Any] = {
"target": mongo_target,
"database": mongo_db,
"collection": collection_name,
"mode": mode,
"documentCount": len(documents),
}
if document_path:
summary["documentPath"] = document_path
if mode == "insert_many":
result = collection.insert_many(documents, ordered=False)
summary["insertedCount"] = len(getattr(result, "inserted_ids", []) or [])
else:
processed = 0
matched_count = 0
modified_count = 0
upserted_ids: list[Any] = []
for document in documents:
key_value = get_field_value(document, upsert_key)
if key_value is None:
raise RuntimeError(f"document is missing UPSERT_KEY={upsert_key}")
result = collection.replace_one({upsert_key: key_value}, document, upsert=True)
processed += 1
matched_count += int(getattr(result, "matched_count", 0) or 0)
modified_count += int(getattr(result, "modified_count", 0) or 0)
upserted_id = getattr(result, "upserted_id", None)
if upserted_id is not None:
upserted_ids.append(json_compatible(upserted_id))
summary["upsertKey"] = upsert_key
summary["upsertedCount"] = processed
summary["matchedCount"] = matched_count
summary["modifiedCount"] = modified_count
if upserted_ids:
summary["upsertedIds"] = upserted_ids
return [
write_json_artifact(
run_id,
node_id,
output_dir,
"mongodb-write-result.json",
summary,
output=output_binding(node_data),
)
]
finally:
close = getattr(client, "close", None)
if callable(close):
close()
def builtin_text_template(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
template_text = params.get("TEMPLATE_TEXT", "").strip()
output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt"
payloads = payloads_from_inputs(inputs)
context = flatten_context(payloads[0] if payloads else {})
rendered = render_template_text(template_text, context)
binding = output_binding(node_data)
manifest = write_json_artifact(
run_id,
node_id,
output_dir,
"rendered-template.json",
{"template": template_text, "context": context, "outputName": output_name},
output=binding,
)
return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)]
def builtin_csv_to_json(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
payload: dict[str, Any]
if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES:
delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()]
columns, rows = load_tabular_rows(source_path, delimiter)
payload = {
"sourcePath": str(source_path),
"columns": columns,
"items": rows,
"rowCount": len(rows),
}
else:
for artifact in inputs:
preview = artifact_preview_payload(artifact) or {}
if preview.get("kind") == "table":
payload = {
"sourcePath": str(artifact.get("path", "")),
"columns": preview.get("columns", []),
"items": preview.get("rows", []),
"rowCount": preview.get("rowCount", len(preview.get("rows", []))),
"previewTruncated": preview.get("truncated", False),
}
return [
write_json_artifact(
run_id,
node_id,
output_dir,
"table.json",
payload,
output=output_binding(node_data),
)
]
raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件")
return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))]
def builtin_json_to_csv(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
rows = rows_from_payloads(payloads_from_inputs(inputs))
binding = output_binding(node_data)
summary = write_json_artifact(
run_id,
node_id,
output_dir,
"csv-export-summary.json",
{"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})},
output=binding,
)
return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)]
def builtin_directory_scan(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
if source_path is not None:
root = source_path if source_path.is_dir() else source_path.parent
payload = {
"sourcePath": str(source_path),
"scanRoot": str(root),
"summary": summarize_directory_tree(root),
}
elif inputs:
payload = {
"sourcePath": "",
"scanRoot": "",
"summary": summarize_input_artifacts(inputs),
}
else:
raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录")
return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))]
def builtin_xplatform_index(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payload = load_xplatform_index_payload(hyperparams(node_data), inputs, output_dir)
return [write_json_artifact(run_id, node_id, output_dir, "dataset-index.json", payload, output=output_binding(node_data))]
def builtin_collection_validate(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
require_intrinsics = parse_bool_param(params, "REQUIRE_INTRINSICS", False)
payload = first_payload_with_keys(inputs, "collections", "items")
if payload is None:
payload = load_xplatform_index_payload(params, inputs, output_dir)
collections = []
valid_collection_count = 0
for collection in payload.get("collections", []):
if not isinstance(collection, dict):
continue
missing_required = list(collection.get("missingRequiredFiles", []))
if require_intrinsics and not collection.get("hasIntrinsics", False):
missing_required.append("intrinsics.json")
valid = len(missing_required) == 0 and int(collection.get("videoCount", 0) or 0) > 0
if valid:
valid_collection_count += 1
collections.append({**collection, "missingRequiredFiles": missing_required, "valid": valid})
items = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
collection_id = str(item.get("collectionId", "")).strip()
items.append({**item, "collectionValid": any(c.get("id") == collection_id and c.get("valid", False) for c in collections)})
result = {
**payload,
"datasetType": "x-platform-validation",
"collections": collections,
"items": items,
"validCollectionCount": valid_collection_count,
"invalidCollectionCount": max(len(collections) - valid_collection_count, 0),
}
return [write_json_artifact(run_id, node_id, output_dir, "dataset-validation.json", result, output=output_binding(node_data))]
def builtin_video_quality(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
min_clarity = parse_float_param(params, "MIN_CLARITY", 0.8)
max_shake = parse_float_param(params, "MAX_SHAKE", 0.1)
max_hand_occlusion = parse_float_param(params, "MAX_HAND_OCCLUSION", 0.1)
clarity_scale = parse_float_param(params, "CLARITY_SCALE", 35.0)
payload = first_payload_with_keys(inputs, "items")
if payload is None:
payload = load_xplatform_index_payload(params, inputs, output_dir)
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
items = []
passed_count = 0
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
source_path = source_path_from_item(item, str(item.get("filename", "video.mp4")), output_dir, archive_sources)
metrics = analyze_video_quality_metrics(source_path, clarity_scale)
passed = (
float_value(metrics.get("clarity"), 0.0) >= min_clarity
and float_value(metrics.get("shake"), 1.0) <= max_shake
and float_value(metrics.get("handOcclusion"), 1.0) <= max_hand_occlusion
)
if passed:
passed_count += 1
items.append(
{
**item,
"quality": {
**metrics,
"passed": passed,
"thresholds": {
"minClarity": min_clarity,
"maxShake": max_shake,
"maxHandOcclusion": max_hand_occlusion,
},
},
}
)
result = {
**payload,
"datasetType": "x-platform-quality",
"items": items,
"videoCount": len(items),
"passedCount": passed_count,
"failedCount": max(len(items) - passed_count, 0),
}
return [write_json_artifact(run_id, node_id, output_dir, "quality-report.json", result, output=output_binding(node_data))]
def builtin_dataset_filter(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "items")
if payload is None:
raise RuntimeError("数据过滤需要上游数据清单输入")
date_start = normalize_date_value(params.get("DATE_START", ""))
date_end = normalize_date_value(params.get("DATE_END", ""))
place = params.get("PLACE", "").strip()
collection_type = params.get("TYPE", "").strip()
min_clarity = parse_float_param(params, "MIN_CLARITY", 0.0)
max_shake = parse_float_param(params, "MAX_SHAKE", 1.0)
max_hand_occlusion = parse_float_param(params, "MAX_HAND_OCCLUSION", 1.0)
filtered = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
meta = item.get("meta") if isinstance(item.get("meta"), dict) else {}
quality = item.get("quality") if isinstance(item.get("quality"), dict) else {}
if place and str(meta.get("place", "")).strip() != place:
continue
if collection_type and str(meta.get("type", "")).strip() != collection_type:
continue
if date_start or date_end:
if not date_in_range(meta.get("date", ""), date_start, date_end):
continue
if float_value(quality.get("clarity"), 0.0) < min_clarity:
continue
if float_value(quality.get("shake"), 1.0) > max_shake:
continue
if float_value(quality.get("handOcclusion"), 1.0) > max_hand_occlusion:
continue
if not bool(quality.get("passed", False)):
continue
filtered.append(item)
result = {
"datasetType": "x-platform-filtered",
"criteria": {
"dateStart": date_start,
"dateEnd": date_end,
"place": place,
"type": collection_type,
"minClarity": min_clarity,
"maxShake": max_shake,
"maxHandOcclusion": max_hand_occlusion,
},
"count": len(filtered),
"items": filtered,
}
if isinstance(payload.get("archiveSources"), dict):
result["archiveSources"] = payload["archiveSources"]
return [write_json_artifact(run_id, node_id, output_dir, "filtered-videos.json", result, output=output_binding(node_data))]
def builtin_video_slicing(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "items")
if payload is None:
raise RuntimeError("视频切片需要上游过滤结果输入")
segment_duration = max(1, parse_int_param(params, "SEGMENT_DURATION_SECONDS", 300))
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
clips: list[dict[str, Any]] = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
source_filename = str(item.get("filename", "")).strip() or "video.mp4"
source_path = source_path_from_item(item, source_filename, output_dir, archive_sources)
metrics = inspect_video_file(source_path)
duration = float(item.get("duration") or metrics.get("duration") or 0.0)
if duration <= 0:
duration = float(metrics.get("duration") or 0.0)
if duration <= 0:
duration = float(segment_duration)
clip_count = max(1, int(math.ceil(duration / segment_duration)))
stem = pathlib.Path(source_filename).stem
clip_specs: list[dict[str, Any]] = []
for clip_index in range(clip_count):
clip_start = clip_index * segment_duration
clip_end = min(duration, clip_start + segment_duration)
clip_name = f"{stem}_clip{clip_index + 1:03d}.mp4"
clip_path = output_dir / "clips" / clip_name
clip_specs.append(
{
"clipIndex": clip_index,
"clipStart": clip_start,
"clipEnd": clip_end,
"clipName": clip_name,
"clipPath": clip_path,
"destination": clip_path,
"startSecond": clip_start,
"endSecond": clip_end,
}
)
clip_metrics_list: list[dict[str, Any]] | None = None
if ffmpeg_available():
try:
log(f"[operator-runner][info] video_slicing uses ffmpeg for {source_filename} ({clip_count} clip(s))")
clip_metrics_list = []
for clip_spec in clip_specs:
clip_metrics_list.append(
{
**clip_spec,
**write_video_segment_with_ffmpeg(
source_path,
pathlib.Path(clip_spec["clipPath"]),
float(clip_spec["clipStart"]),
float(clip_spec["clipEnd"]),
),
}
)
except RuntimeError as exc:
log(f"[operator-runner][warn] ffmpeg video slicing failed for {source_filename}, fallback to opencv: {exc}")
clip_metrics_list = None
if clip_metrics_list is None:
if len(clip_specs) > 1:
clip_metrics_list = write_video_segments(source_path, clip_specs)
else:
only = clip_specs[0]
clip_metrics_list = [
{
**only,
**write_video_segment(source_path, only["clipPath"], float(only["clipStart"]), float(only["clipEnd"])),
}
]
for clip_spec in clip_metrics_list:
clip_start = float(clip_spec["clipStart"])
clip_end = float(clip_spec["clipEnd"])
clip_name = str(clip_spec["clipName"])
clip_path = pathlib.Path(clip_spec["clipPath"])
clips.append(
{
"clipId": pathlib.Path(clip_name).stem,
"clipName": clip_name,
"clipPath": str(clip_path),
"sourceVideo": source_filename,
"sourcePath": str(source_path),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
"collectionId": item.get("collectionId", ""),
"meta": item.get("meta", {}),
"quality": item.get("quality", {}),
"startSecond": round(clip_start, 3),
"endSecond": round(clip_end, 3),
"duration": clip_spec.get("duration", round(max(clip_end - clip_start, 0.0), 3)),
"resolution": clip_spec.get("resolution"),
"frameRate": clip_spec.get("frameRate"),
}
)
manifest = {
"datasetType": "x-platform-clips",
"segmentDurationSeconds": segment_duration,
"clipCount": len(clips),
"items": clips,
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "clips-manifest.json", manifest, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "clips", node_data=node_data, output=output_binding(node_data)))
return artifacts
def builtin_video_annotation(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "clipCount", "items")
if payload is None:
raise RuntimeError("视频标注需要上游切片结果输入")
frame_interval = max(0.1, parse_float_param(params, "FRAME_INTERVAL_SECONDS", 1.0))
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
items = []
for clip in payload.get("items", []):
if not isinstance(clip, dict):
continue
clip_name = str(clip.get("clipName", "")).strip() or pathlib.Path(str(clip.get("clipPath", ""))).name
artifact = clip_artifacts.get(clip_name)
clip_path: pathlib.Path | None = None
if artifact is not None:
try:
clip_path = source_path_from_item(
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
clip_name,
output_dir,
archive_sources,
)
except RuntimeError:
clip_path = None
if clip_path is None:
direct_clip_path = str(clip.get("clipPath", "")).strip()
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
clip_path = pathlib.Path(direct_clip_path)
if clip_path is None:
clip_path = materialize_clip_from_source(clip, output_dir, archive_sources)
if clip_path is None or not clip_path.exists():
raise RuntimeError(f"无法获取真实输入文件: {clip_name}")
annotation = annotate_video_clip(clip_path, frame_interval)
annotation_filename = f"{pathlib.Path(clip_name).stem}_annotation.json"
annotation_payload = {
"clipId": clip.get("clipId", pathlib.Path(clip_name).stem),
"sourceVideo": clip.get("sourceVideo", ""),
**annotation,
}
items.append({**clip, "annotation": annotation_payload, "annotationFile": annotation_filename})
write_json_artifact(
run_id,
node_id,
output_dir / "annotations",
annotation_filename,
annotation_payload,
output=output_binding(node_data),
)
manifest = {
"datasetType": "x-platform-annotations",
"annotationCount": len(items),
"items": items,
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "annotations-manifest.json", manifest, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "annotations", node_data=node_data, output=output_binding(node_data)))
return artifacts
def builtin_result_package(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payload = first_payload_with_keys(inputs, "annotationCount", "items")
if payload is None:
raise RuntimeError("成品打包需要上游标注结果输入")
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
packages = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
clip_name = str(item.get("clipName", "")).strip()
clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem
package_name = f"{clip_id}_package"
package_dir = output_dir / "packages" / package_name
package_dir.mkdir(parents=True, exist_ok=True)
artifact = clip_artifacts.get(clip_name)
clip_path: pathlib.Path | None = None
if artifact is not None:
try:
clip_path = source_path_from_item(
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
clip_name,
output_dir,
archive_sources,
)
except RuntimeError:
clip_path = None
if clip_path is None:
direct_clip_path = str(item.get("clipPath", "")).strip()
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
clip_path = pathlib.Path(direct_clip_path)
if clip_path is None:
clip_path = materialize_clip_from_source(item, output_dir, archive_sources)
if clip_path is not None and clip_path.exists():
shutil.copy2(clip_path, package_dir / clip_name)
quality_payload = {
"clip_id": clip_id,
"source_video": item.get("sourceVideo", ""),
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
}
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
config_payload = package_config_payload(item, payload)
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_quality.json", quality_payload, output=output_binding(node_data))
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation_payload, output=output_binding(node_data))
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_config.json", config_payload, output=output_binding(node_data))
package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()]
packages.append(
{
"packageName": package_name,
"clipId": clip_id,
"clipName": clip_name,
"sourceVideo": item.get("sourceVideo", ""),
"clipDuration": item.get("duration", 0),
"duration": item.get("duration", 0),
"startSecond": item.get("startSecond"),
"endSecond": item.get("endSecond"),
"collectionId": item.get("collectionId", ""),
"meta": item.get("meta", {}),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
"fileCount": len(package_files),
"pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1"],
"storagePath": str(package_dir),
"quality": quality_payload,
"annotation": annotation_payload,
}
)
manifest = {
"datasetType": "x-platform-packages",
"packageCount": len(packages),
"items": packages,
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "packages-manifest.json", manifest, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "packages", node_data=node_data, output=output_binding(node_data)))
return artifacts
def builtin_audit_sampling(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "packageCount", "items")
if payload is None:
raise RuntimeError("抽检采样需要上游成品数据输入")
rules = parse_json_param(params, "RULES_JSON", [])
sampled_items = []
summaries = []
for rule in rules if isinstance(rules, list) else []:
if not isinstance(rule, dict):
continue
rule_id = str(rule.get("id", "rule")).strip() or "rule"
field_path = str(rule.get("field", "")).strip()
operator = str(rule.get("operator", "lt")).strip() or "lt"
value = rule.get("value")
percent = float(rule.get("samplePercent", 10) or 0)
matched = []
sampled = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
if not compare_values(get_field_value(item, field_path), operator, value):
continue
matched.append(item)
package_name = str(item.get("packageName", "")).strip() or str(item.get("clipId", "")).strip()
if stable_sample_hit(f"{rule_id}:{package_name}", percent):
sampled.append(item)
sampled_items.append({**item, "sampleRuleId": rule_id})
summaries.append(
{
"id": rule_id,
"field": field_path,
"operator": operator,
"value": value,
"samplePercent": percent,
"matchedCount": len(matched),
"sampledCount": len(sampled),
}
)
manifest = {
"datasetType": "x-platform-audit-samples",
"rules": summaries,
"sampledCount": len(sampled_items),
"items": sampled_items,
}
return [write_json_artifact(run_id, node_id, output_dir, "audit-samples.json", manifest, output=output_binding(node_data))]
def builtin_condition_router(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_path = params.get("FIELD_PATH", "").strip()
operator = params.get("OPERATOR", "eq").strip().lower() or "eq"
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
rows = collect_items(payloads_from_inputs(inputs))
matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)]
outputs = node_data.get("outputs", []) or [
{"id": "matched", "name": "matched", "type": "json"},
{"id": "unmatched", "name": "unmatched", "type": "json"},
]
return [
write_json_artifact(
run_id,
node_id,
output_dir,
f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json",
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)},
str(outputs[0].get("type", "json")),
output=outputs[0],
),
write_json_artifact(
run_id,
node_id,
output_dir,
f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json",
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)},
str(outputs[1].get("type", "json")),
output=outputs[1],
),
]
def builtin_lerobot_summary(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
if source_path is not None:
root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent)
payload = summarize_lerobot_root(root)
payload["sourcePath"] = str(source_path)
elif inputs:
payload = summarize_lerobot_from_inputs(inputs)
payload["sourcePath"] = ""
else:
raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录")
payload["datasetType"] = "lerobot"
return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))]
def builtin_rosbag_summary(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
if source_path is not None:
if source_path.is_dir():
payload = {
"datasetType": "rosbag2",
"sourcePath": str(source_path),
"summary": summarize_directory_tree(source_path),
}
metadata_yaml = source_path / "metadata.yaml"
if metadata_yaml.exists():
payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"]
db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()]
if db3_files:
payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"]
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
payload = summarize_rosbag_file(source_path)
payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag")
payload["sourcePath"] = str(source_path)
else:
for artifact in inputs:
preview = artifact_preview_payload(artifact) or {}
if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}:
payload = {
"datasetType": preview.get("format", "rosbag"),
"sourcePath": str(artifact.get("path", "")),
"preview": preview,
}
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件")
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
def builtin_generic(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_label: str,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = [load_artifact_payload(item) for item in inputs]
summary = {
"nodeLabel": node_label,
"inputCount": len(inputs),
"inputs": [summarize_json_like(item) for item in payloads],
"config": {
"bucket": node_data.get("bucket", ""),
"prefix": node_data.get("prefix", ""),
"pattern": node_data.get("pattern", ""),
},
}
outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}]
artifacts: list[dict[str, Any]] = []
for index, output in enumerate(outputs):
output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}"
filename = f"{output_name}.json"
payload = {**summary, "outputName": output_name}
artifacts.append(
write_json_artifact(
run_id,
node_id,
output_dir,
filename,
payload,
str(output.get("type", "json")),
output=output,
)
)
return artifacts
def execute_command(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
command: str,
) -> list[dict[str, Any]]:
completed = subprocess.run(
command,
cwd=output_dir,
shell=True,
text=True,
capture_output=True,
env=os.environ.copy(),
check=False,
)
for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)):
for line in content.splitlines():
if line.strip():
log(f"[operator-runner][command][{stream_name}] {line}")
if completed.returncode != 0:
raise RuntimeError(f"command failed with exit code {completed.returncode}")
artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data)
if artifacts:
return artifacts
payload = {"stdout": completed.stdout, "stderr": completed.stderr}
return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))]
def select_handler(node_label: str) -> str:
normalized = node_label.lower()
if any(token in normalized for token in ("mongodb", "mongo 写入", "mongo write", "写入 mongo")):
return "mongodb_write"
if any(token in normalized for token in ("x 平台数据集索引", "x-platform", "数据集索引", "dataset index")):
return "xplatform_index"
if any(token in normalized for token in ("采集目录校验", "目录校验", "validate collection")):
return "collection_validate"
if any(token in normalized for token in ("视频质检", "quality check", "quality")):
return "video_quality"
if any(token in normalized for token in ("数据过滤", "dataset filter", "video filter")):
return "dataset_filter"
if any(token in normalized for token in ("视频切片", "video slicing", "slice")):
return "video_slicing"
if any(token in normalized for token in ("视频标注", "annotation", "annotate")):
return "video_annotation"
if any(token in normalized for token in ("成品打包", "package output", "package result")):
return "result_package"
if any(token in normalized for token in ("抽检采样", "audit sample", "sampling")):
return "audit_sampling"
if "lerobot" in normalized:
return "lerobot"
if any(token in normalized for token in ("rosbag", "mcap", "ros bag")):
return "rosbag"
if any(token in normalized for token in ("csv 转 json", "csv转json", "csv to json", "tsv 转 json", "tsv转json", "tsv to json")):
return "csv_to_json"
if any(token in normalized for token in ("json 转 csv", "json转csv", "json to csv")):
return "json_to_csv"
if any(token in normalized for token in ("字段提取", "json提取", "json 提取", "extract")):
return "json_extract"
if any(token in normalized for token in ("json过滤", "json 过滤", "filter")):
return "json_filter"
if any(token in normalized for token in ("json映射", "json 映射", "rename", "映射重命名")):
return "json_map"
if any(token in normalized for token in ("模板渲染", "template")):
return "text_template"
if any(token in normalized for token in ("条件分流", "conditional", "router", "route")):
return "condition_router"
if any(token in normalized for token in ("目录扫描", "directory", "browse", "browse dir")):
return "directory_scan"
if any(token in normalized for token in ("merge", "合并")):
return "merge"
if any(token in normalized for token in ("intersect", "交集")):
return "intersection"
if any(token in normalized for token in ("distribution", "分配")):
return "distribution"
if any(token in normalized for token in ("upload", "export", "上传", "导出")):
return "upload"
if any(token in normalized for token in ("source", "scanner", "oss", "scan", "扫描", "数据源")):
return "source"
return "generic"
def main() -> int:
run_id = os.getenv("RUN_ID", "").strip()
workflow_id = os.getenv("WORKFLOW_ID", "").strip()
node_id = os.getenv("NODE_ID", "").strip()
node_label = os.getenv("NODE_LABEL", "").strip() or node_id
output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id)))
output_dir.mkdir(parents=True, exist_ok=True)
node_data = load_json_env("NODE_DATA_JSON", {})
inputs = load_input_artifacts()
started_at = now_iso()
start_cpu, start_memory = resource_snapshot()
logs: list[dict[str, Any]] = []
def append_log(level: str, message: str) -> None:
logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label})
log(f"[operator-runner][{level.lower()}] {message}")
append_log("INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir}")
append_log("INFO", "Preparing operator execution context")
params = hyperparams(node_data)
command = params.get("COMMAND") or params.get("SCRIPT")
artifacts: list[dict[str, Any]]
if command:
append_log("INFO", f"Executing explicit command: {command}")
artifacts = execute_command(run_id, node_id, output_dir, node_data, command)
else:
handler = select_handler(node_label)
append_log("INFO", f"Selected built-in handler: {handler}")
if handler == "source":
artifacts = builtin_source(run_id, node_id, output_dir, node_data, inputs)
elif handler == "merge":
artifacts = builtin_merge(run_id, node_id, output_dir, node_data, inputs)
elif handler == "intersection":
artifacts = builtin_intersection(run_id, node_id, output_dir, node_data, inputs)
elif handler == "distribution":
artifacts = builtin_distribution(run_id, node_id, output_dir, node_data, inputs)
elif handler == "json_extract":
artifacts = builtin_json_extract(run_id, node_id, output_dir, node_data, inputs)
elif handler == "json_filter":
artifacts = builtin_json_filter(run_id, node_id, output_dir, node_data, inputs)
elif handler == "json_map":
artifacts = builtin_json_map(run_id, node_id, output_dir, node_data, inputs)
elif handler == "mongodb_write":
artifacts = builtin_mongodb_write(run_id, node_id, output_dir, node_data, inputs)
elif handler == "text_template":
artifacts = builtin_text_template(run_id, node_id, output_dir, node_data, inputs)
elif handler == "csv_to_json":
artifacts = builtin_csv_to_json(run_id, node_id, output_dir, node_data, inputs)
elif handler == "json_to_csv":
artifacts = builtin_json_to_csv(run_id, node_id, output_dir, node_data, inputs)
elif handler == "directory_scan":
artifacts = builtin_directory_scan(run_id, node_id, output_dir, node_data, inputs)
elif handler == "xplatform_index":
artifacts = builtin_xplatform_index(run_id, node_id, output_dir, node_data, inputs)
elif handler == "collection_validate":
artifacts = builtin_collection_validate(run_id, node_id, output_dir, node_data, inputs)
elif handler == "video_quality":
artifacts = builtin_video_quality(run_id, node_id, output_dir, node_data, inputs)
elif handler == "dataset_filter":
artifacts = builtin_dataset_filter(run_id, node_id, output_dir, node_data, inputs)
elif handler == "video_slicing":
artifacts = builtin_video_slicing(run_id, node_id, output_dir, node_data, inputs)
elif handler == "video_annotation":
artifacts = builtin_video_annotation(run_id, node_id, output_dir, node_data, inputs)
elif handler == "result_package":
artifacts = builtin_result_package(run_id, node_id, output_dir, node_data, inputs)
elif handler == "audit_sampling":
artifacts = builtin_audit_sampling(run_id, node_id, output_dir, node_data, inputs)
elif handler == "condition_router":
artifacts = builtin_condition_router(run_id, node_id, output_dir, node_data, inputs)
elif handler == "lerobot":
artifacts = builtin_lerobot_summary(run_id, node_id, output_dir, node_data, inputs)
elif handler == "rosbag":
artifacts = builtin_rosbag_summary(run_id, node_id, output_dir, node_data, inputs)
elif handler == "upload":
artifacts = builtin_upload(run_id, node_id, output_dir, node_data, inputs)
else:
artifacts = builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs)
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
result = {
"logs": logs,
"artifacts": artifacts,
"resourceSeries": resource_series(start_cpu, start_memory, started_at),
}
stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip()
if stage_manifest_path:
write_json_file(stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
else:
write_json_file(default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip()
if result_output_path:
written_path = write_json_file(result_output_path, result)
print(RESULT_PATH_PREFIX + str(written_path), flush=True)
return 0
default_result_path = write_json_file(default_result_output_path(output_dir), result)
print(RESULT_PATH_PREFIX + str(default_result_path), flush=True)
return 0
def tempfile_dir(run_id: str, node_id: str) -> str:
base = pathlib.Path("/tmp/droboticflow")
return str(base / (run_id or "run") / (node_id or "node"))
if __name__ == "__main__":
try:
sys.exit(main())
except Exception as error: # pragma: no cover
log(f"[operator-runner][error] {error}")
sys.exit(1)