#!/usr/bin/env python3 from __future__ import annotations import datetime as dt import contextlib import fnmatch import hashlib import importlib import json import mimetypes import os import pathlib import base64 import csv import math import re import shutil import sqlite3 import subprocess import sys import tarfile import time import zipfile from typing import Any from urllib import error, request from urllib.parse import urlparse PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[2] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) RUNNER_ROOT = pathlib.Path(__file__).resolve().parent if str(RUNNER_ROOT) not in sys.path: sys.path.insert(0, str(RUNNER_ROOT)) from resource_monitor import ResourceSampler, resource_snapshot RESULT_PREFIX = "DROBOTICFLOW_RESULT=" RESULT_PATH_PREFIX = "DROBOTICFLOW_RESULT_PATH=" TEXT_FILE_SUFFIXES = { ".txt", ".log", ".md", ".yaml", ".yml", ".xml", ".html", ".htm", } TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"} IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"} VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"} AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"} ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"} ARCHIVE_FILE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip") XPLATFORM_VIDEO_META_FILES = ("video_meta.json", "video_metadata.json") XPLATFORM_INLINE_JSON_FILES = {"meta.json", "intrinsics.json", *XPLATFORM_VIDEO_META_FILES} PREVIEW_TEXT_LIMIT = 12_000 PREVIEW_TEXT_LINE_LIMIT = 240 PREVIEW_TABLE_ROW_LIMIT = 24 PREVIEW_FILE_LIST_LIMIT = 40 PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024 DATASET_PARTITION_SIZE = 500 QUALITY_SAMPLE_LIMIT = 48 QUALITY_ANALYSIS_MAX_DIMENSION = 640 QUALITY_SKIN_COMPONENT_MIN_RATIO = 0.002 DEFAULT_MIN_CLARITY = 0.8 DEFAULT_MAX_SHAKE = 0.1 DEFAULT_MAX_HAND_OCCLUSION = 0.1 TRUE_ENV_VALUES = {"1", "true", "yes", "on"} FALSE_ENV_VALUES = {"0", "false", "no", "off"} MAX_SAFE_INTEGER = 9_007_199_254_740_991 EXACT_POSITIVE_INT_RE = re.compile(r"^[1-9][0-9]*$") INTEGERISH_RE = re.compile(r"^[+-]?[0-9]+$") SUPPORTED_COMPARISON_OPERATORS = {"contains", "eq", "exists", "gt", "gte", "in", "lt", "lte", "missing", "ne", "regex"} VALID_SOURCE_DATA_SOURCES = {"INLINE", "LOCAL", "BOS", "UPSTREAM"} RESERVED_MONGO_COLLECTIONS = { "projects", "operators", "workflows", "templates", "runs", "sourceStates", "auditTasks", "samplingTasks", "users", "storageConnections", "settings", } def env_bool(name: str, default: bool) -> bool: value = os.getenv(name, "").strip().lower() if value == "": return default if value in TRUE_ENV_VALUES: return True if value in FALSE_ENV_VALUES: return False raise RuntimeError(f"{name} must be one of true/false/1/0/yes/no/on/off") def compile_policy_regex(name: str) -> re.Pattern[str] | None: value = os.getenv(name, "").strip() if not value: return None try: return re.compile(value) except re.error as error: raise RuntimeError(f"{name} is not a valid regular expression: {error}") from error def validate_command_policy(command: str) -> None: if not env_bool("OPERATOR_ALLOW_SHELL_COMMANDS", False): raise RuntimeError("explicit shell commands are disabled by OPERATOR_ALLOW_SHELL_COMMANDS") denylist = compile_policy_regex("OPERATOR_COMMAND_DENYLIST_REGEX") if denylist and denylist.search(command): raise RuntimeError("explicit shell command was rejected by OPERATOR_COMMAND_DENYLIST_REGEX") allowlist = compile_policy_regex("OPERATOR_COMMAND_ALLOWLIST_REGEX") if allowlist and not allowlist.search(command): raise RuntimeError("explicit shell command was not allowed by OPERATOR_COMMAND_ALLOWLIST_REGEX") DATASET_PARTITION_KEYS = ( "items", "entries", "rows", "collections", "catalogCollections", "catalogItems", "catalogEntries", ) PAYLOAD_FILE_READ_ATTEMPTS = 60 PAYLOAD_FILE_READ_DELAY_SECONDS = 1.0 def now_iso() -> str: return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") def log(message: str) -> None: print(message, flush=True) def load_json_env(name: str, default: Any) -> Any: value = os.getenv(name, "").strip() if not value: return default return json.loads(value) def load_json_file(path: str, default: Any) -> Any: raw = str(path or "").strip() if not raw: return default candidate = pathlib.Path(raw) if not candidate.exists(): return default return json.loads(candidate.read_text(encoding="utf-8")) def write_json_file(path: str | pathlib.Path, value: Any) -> pathlib.Path: candidate = pathlib.Path(path) candidate.parent.mkdir(parents=True, exist_ok=True) candidate.write_text(json.dumps(json_compatible(value), ensure_ascii=False), encoding="utf-8") return candidate def text_value(value: Any) -> str: if isinstance(value, bytes): return value.decode("utf-8", errors="replace") return str(value or "") def json_compatible(value: Any) -> Any: if isinstance(value, bytes): return value.decode("utf-8", errors="replace") if isinstance(value, dict): return {str(key): json_compatible(item) for key, item in value.items()} if isinstance(value, list): return [json_compatible(item) for item in value] if isinstance(value, tuple): return [json_compatible(item) for item in value] return value def load_input_artifacts() -> list[dict[str, Any]]: input_artifacts_path = os.getenv("INPUT_ARTIFACTS_PATH", "").strip() if input_artifacts_path: loaded = load_json_file(input_artifacts_path, []) if isinstance(loaded, list): return json_compatible(loaded) loaded = load_json_env("INPUT_ARTIFACTS_JSON", []) if isinstance(loaded, list): return json_compatible(loaded) return [] def load_input_bindings() -> list[dict[str, Any]]: input_bindings_path = os.getenv("INPUT_BINDINGS_PATH", "").strip() if input_bindings_path: loaded = load_json_file(input_bindings_path, []) if isinstance(loaded, list): return json_compatible(loaded) loaded = load_json_env("INPUT_BINDINGS_JSON", []) if isinstance(loaded, list): return json_compatible(loaded) return [] def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None: for key in keys: value = payload.get(key) if isinstance(value, bool): continue if isinstance(value, (int, float)) and value >= 0: return value return None def _metric_volume(payloads: list[dict[str, Any]]) -> dict[str, int | float]: candidates: dict[str, list[int | float]] = { "trajectoryCount": [], "durationSeconds": [], "bytes": [], "fileCount": [], } for payload in payloads: data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {} trajectory_count = _first_numeric( payload, ("trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"), ) if trajectory_count is None and data_ref: trajectory_count = _first_numeric(data_ref, ("itemCount",)) duration_seconds = _first_numeric(payload, ("durationSeconds", "totalDurationSeconds", "totalDuration", "duration")) total_bytes = _first_numeric(payload, ("totalBytes", "bytes", "sizeBytes")) file_count = _first_numeric(payload, ("fileCount", "filesCount")) if trajectory_count is not None: candidates["trajectoryCount"].append(int(trajectory_count)) if duration_seconds is not None: candidates["durationSeconds"].append(float(duration_seconds)) if total_bytes is not None: candidates["bytes"].append(int(total_bytes)) if file_count is not None: candidates["fileCount"].append(int(file_count)) return {key: max(values) for key, values in candidates.items() if values} def _artifact_payloads(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]: return [item["data"] for item in artifacts if isinstance(item.get("data"), dict)] def _dataset_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]: refs: list[dict[str, Any]] = [] seen: set[str] = set() for artifact in artifacts: payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {} if not data_ref: continue item = { "name": text_value(artifact.get("name")), "kind": text_value(data_ref.get("kind")), "datasetType": text_value(data_ref.get("datasetType")), "manifestPath": text_value(data_ref.get("manifestPath")), "payloadPath": text_value(data_ref.get("payloadPath")), "partitionDir": text_value(data_ref.get("partitionDir")), "partitionCount": int(data_ref.get("partitionCount") or 0), "itemCount": int(data_ref.get("itemCount") or 0), "datasetId": text_value(data_ref.get("datasetId")), "datasetVersionId": text_value(data_ref.get("datasetVersionId")), "split": text_value(data_ref.get("split")), "fingerprint": text_value(data_ref.get("fingerprint")), } key = "|".join( text_value(item.get(name)) for name in ("datasetVersionId", "datasetId", "manifestPath", "payloadPath", "partitionDir", "split") ) if key in seen: continue seen.add(key) refs.append(item) return refs def infer_data_metrics( node_data: dict[str, Any], artifacts: list[dict[str, Any]], inputs: list[dict[str, Any]], ) -> dict[str, Any] | None: stage = text_value(node_data.get("businessStage")).strip().lower() if stage not in {"collect", "process", "clean", "validate", "annotate", "audit", "train"}: return None if node_data.get("metricBoundary") is False: return None output_payloads = _artifact_payloads(artifacts) for payload in output_payloads: explicit = payload.get("dataMetrics") if isinstance(explicit, dict): normalized = json_compatible(explicit) normalized.setdefault("schemaVersion", 1) normalized.setdefault("stage", stage) normalized.setdefault("quality", "exact") normalized.setdefault("occurredAt", now_iso()) return normalized input_payloads = _artifact_payloads(inputs) input_volume = _metric_volume(input_payloads) output_volume = _metric_volume(output_payloads) metrics: dict[str, Any] = { "schemaVersion": 1, "stage": stage, "input": input_volume, "output": output_volume, "datasetRefs": _dataset_refs(artifacts), "quality": "derived", "occurredAt": now_iso(), } if stage == "collect" and not input_volume: metrics.pop("input") if stage == "validate": valid = any(payload.get("valid") is True for payload in output_payloads) metrics["technicalValid"] = output_volume if valid else {} if not valid and input_volume: metrics["rejected"] = input_volume if stage in {"annotate", "audit"}: metrics["trainReady"] = output_volume if stage == "clean" and input_volume and output_volume: rejected: dict[str, int | float] = {} for key in ("trajectoryCount", "durationSeconds", "bytes", "fileCount"): if key in input_volume and key in output_volume: rejected[key] = max(0, input_volume[key] - output_volume[key]) if rejected: metrics["rejected"] = rejected if stage == "train": training_inputs: list[dict[str, Any]] = [] for ref in _dataset_refs(inputs): training_inputs.append( { "datasetId": ref.get("datasetId", ""), "datasetVersionId": ref.get("datasetVersionId", ""), "name": ref.get("name", ""), "split": ref.get("split", ""), "volume": input_volume, "manifestPath": ref.get("manifestPath", ""), "partitionDir": ref.get("partitionDir", ""), "external": not bool(ref.get("datasetVersionId")), "quality": "derived", } ) metrics["trainingInputs"] = training_inputs return metrics def stage_manifest_payload( run_id: str, workflow_id: str, node_id: str, node_label: str, output_dir: pathlib.Path, artifacts: list[dict[str, Any]], data_metrics: dict[str, Any] | None = None, ) -> dict[str, Any]: manifest_artifacts: list[dict[str, Any]] = [] data_refs: list[dict[str, Any]] = [] for artifact in artifacts: item = { "id": text_value(artifact.get("id")), "name": text_value(artifact.get("name")), "type": text_value(artifact.get("type")), "path": text_value(artifact.get("path")), "size": text_value(artifact.get("size")), } if text_value(artifact.get("portId")): item["portId"] = text_value(artifact.get("portId")) if text_value(artifact.get("portName")): item["portName"] = text_value(artifact.get("portName")) if isinstance(artifact.get("metadata"), dict) and artifact["metadata"]: item["metadata"] = json_compatible(artifact["metadata"]) data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {} if data_ref: item["dataRef"] = json_compatible(data_ref) data_refs.append( { "artifactId": item["id"], "artifactName": item["name"], "portId": item.get("portId", ""), **json_compatible(data_ref), } ) manifest_artifacts.append(item) result = { "runId": run_id, "workflowId": workflow_id, "nodeId": node_id, "nodeLabel": node_label, "outputDir": str(output_dir), "artifactCount": len(manifest_artifacts), "artifacts": manifest_artifacts, "dataRefs": data_refs, "primaryDataRefs": data_refs, "generatedAt": now_iso(), } if data_metrics: result["dataMetrics"] = json_compatible(data_metrics) return result def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path: return output_dir.parent / "result.json" def default_stage_manifest_path(output_dir: pathlib.Path) -> pathlib.Path: return output_dir.parent / "stage-manifest.json" def normalize_json_payload(value: Any) -> dict[str, Any]: if isinstance(value, dict): return json_compatible(value) if isinstance(value, list): return {"items": json_compatible(value), "itemCount": len(value)} return {"value": json_compatible(value)} def bos_prefix(value: str) -> str: return str(value or "").strip().strip("/") def canonical_bos_path(bucket: str, key: str) -> str: normalized_bucket = str(bucket or "").strip() normalized_key = str(key or "").strip().lstrip("/") if not normalized_bucket: return "" if not normalized_key: return f"bos://{normalized_bucket}" return f"bos://{normalized_bucket}/{normalized_key}" def parse_bos_path(value: str) -> tuple[str, str]: raw = str(value or "").strip() if not raw.startswith("bos://"): return "", "" bucket_and_key = raw[len("bos://") :] if not bucket_and_key: return "", "" bucket, _, key = bucket_and_key.partition("/") return bucket.strip(), bos_prefix(key) def is_xplatform_video_meta_name(value: str) -> bool: return str(value or "").strip().lower() in XPLATFORM_VIDEO_META_FILES def load_collection_video_meta(collection_dir: pathlib.Path) -> tuple[dict[str, dict[str, Any]], str]: for candidate in XPLATFORM_VIDEO_META_FILES: payload = optional_json_file(collection_dir / candidate) if payload is not None: return parse_video_meta_rows(payload), candidate return {}, "" def sampled_frame_positions(frame_count: int, sample_limit: int = QUALITY_SAMPLE_LIMIT) -> list[int]: if frame_count <= 0: return [] if frame_count <= sample_limit: return list(range(frame_count)) last_frame = max(frame_count - 1, 0) positions = { min(last_frame, max(0, int(round(index * last_frame / max(sample_limit - 1, 1))))) for index in range(sample_limit) } return sorted(positions) def resize_quality_frame(frame: Any, max_dimension: int = QUALITY_ANALYSIS_MAX_DIMENSION) -> Any: try: height, width = frame.shape[:2] except Exception: return frame longest_edge = max(width, height) if longest_edge <= 0 or longest_edge <= max_dimension: return frame scale = max_dimension / float(longest_edge) target_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale)))) cv2, _ = require_cv2() return cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA) def build_skin_mask(frame: Any) -> Any: cv2, np = require_cv2() ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb) ycrcb_mask = cv2.inRange( ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8), ) hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) hsv_mask = cv2.inRange( hsv, np.array((0, 20, 40), dtype=np.uint8), np.array((25, 220, 255), dtype=np.uint8), ) return cv2.bitwise_and(ycrcb_mask, hsv_mask) def largest_skin_component_ratio(mask: Any, min_component_ratio: float = QUALITY_SKIN_COMPONENT_MIN_RATIO) -> float: cv2, np = require_cv2() if mask is None: return 0.0 total_area = int(mask.shape[0] * mask.shape[1]) if len(mask.shape) >= 2 else 0 if total_area <= 0: return 0.0 kernel = np.ones((5, 5), dtype=np.uint8) cleaned = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel) cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_CLOSE, kernel) component_threshold = max(1, int(total_area * max(min_component_ratio, 0.0))) count, _, stats, _ = cv2.connectedComponentsWithStats(cleaned) largest_area = 0 for index in range(1, count): area = int(stats[index, cv2.CC_STAT_AREA]) if area < component_threshold: continue if area > largest_area: largest_area = area return float(largest_area / total_area) def motion_jitter_ratio(shifts: list[tuple[float, float]], diagonal: float) -> float: if not shifts: return 0.0 _, np = require_cv2() normalized_diagonal = max(diagonal, 1.0) dx_values = [item[0] for item in shifts] dy_values = [item[1] for item in shifts] median_dx = float(np.median(dx_values)) median_dy = float(np.median(dy_values)) jitters = [ math.sqrt((dx - median_dx) ** 2 + (dy - median_dy) ** 2) / normalized_diagonal for dx, dy in shifts ] return sum(jitters) / max(len(jitters), 1) def bos_key_join(prefix: str, relative_path: str) -> str: normalized_prefix = bos_prefix(prefix) normalized_relative = str(relative_path or "").strip().lstrip("/") if not normalized_prefix: return normalized_relative if not normalized_relative: return normalized_prefix return f"{normalized_prefix}/{normalized_relative}" def require_bos_sdk(): try: import baidubce.protocol as protocol # type: ignore from baidubce.auth.bce_credentials import BceCredentials # type: ignore from baidubce.bce_client_configuration import BceClientConfiguration # type: ignore from baidubce.services.bos.bos_client import BosClient # type: ignore except Exception as error: # pragma: no cover - runtime environment specific raise RuntimeError("BOS 扫描需要 bce-python-sdk 运行时") from error return protocol, BceCredentials, BceClientConfiguration, BosClient def build_bos_client(endpoint: str, access_key_id: str, secret_access_key: str): protocol_module, BceCredentials, BceClientConfiguration, BosClient = require_bos_sdk() parsed = urlparse(endpoint if "://" in endpoint else f"https://{endpoint}") scheme = (parsed.scheme or "https").lower() host = parsed.netloc or parsed.path if not host: raise RuntimeError("BOS endpoint 不能为空") config = BceClientConfiguration( credentials=BceCredentials(access_key_id, secret_access_key), endpoint=host, protocol=protocol_module.HTTPS if scheme == "https" else protocol_module.HTTP, ) return BosClient(config), scheme def format_size(num_bytes: int) -> str: if num_bytes < 1024: return f"{num_bytes} B" if num_bytes < 1024 * 1024: return f"{num_bytes / 1024:.1f} KiB" if num_bytes < 1024 * 1024 * 1024: return f"{num_bytes / (1024 * 1024):.1f} MiB" return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB" MAX_SHA256_FILE_BYTES = 64 * 1024 * 1024 def sha256_file(path: pathlib.Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(65536), b""): digest.update(chunk) return digest.hexdigest() def artifact_sha256(path: pathlib.Path, size_bytes: int) -> tuple[str, dict[str, Any]]: if size_bytes > MAX_SHA256_FILE_BYTES: return "", {"sha256Skipped": True, "sha256SkipReason": "size-limit", "sha256SkipBytes": size_bytes} return sha256_file(path), {} def require_cv2() -> tuple[Any, Any]: try: import cv2 # type: ignore import numpy as np # type: ignore except Exception as error: # pragma: no cover - environment specific raise RuntimeError("视频算子需要 opencv-python-headless 和 numpy 运行时") from error return cv2, np def data_url_for_path(path: pathlib.Path, mime_type: str) -> str: encoded = base64.b64encode(path.read_bytes()).decode("ascii") return f"data:{mime_type};base64,{encoded}" def decode_data_url(url: str) -> tuple[str, bytes] | None: if not url.startswith("data:") or "," not in url: return None header, encoded = url.split(",", 1) if ";base64" not in header: return None mime_type = header[5:].split(";", 1)[0] or "application/octet-stream" try: return mime_type, base64.b64decode(encoded) except Exception: return None def download_remote_source_to_path(url: str, path: pathlib.Path) -> pathlib.Path | None: parsed = urlparse(url) if parsed.scheme.lower() not in {"http", "https"}: return None path.parent.mkdir(parents=True, exist_ok=True) remote_request = request.Request( url, headers={ "User-Agent": "DroboticFlow operator-runner/1.0", }, ) try: with request.urlopen(remote_request, timeout=300) as response, path.open("wb") as handle: shutil.copyfileobj(response, handle, length=1024 * 1024) except error.HTTPError as exc: raise RuntimeError(f"下载远程输入失败({exc.code}): {path.name}") from exc except error.URLError as exc: raise RuntimeError(f"下载远程输入失败: {path.name}") from exc return path def download_remote_source(url: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None: path = output_dir / "_materialized_sources" / default_name return download_remote_source_to_path(url, path) def workspace_cached_bos_source_path(raw_path: str, output_dir: pathlib.Path) -> pathlib.Path | None: bucket, key = parse_bos_path(raw_path) if not bucket or not key: return None workspace_root = str(os.getenv("WORKSPACE_ROOT", "")).strip() if workspace_root: root = pathlib.Path(workspace_root) if root.is_absolute(): return root / "_bos_cache" / bucket / key return output_dir / "_materialized_sources" / bucket / key def download_bos_source(raw_path: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None: target = workspace_cached_bos_source_path(raw_path, output_dir) if target is None: return None if target.exists() and target.stat().st_size > 0: return target bucket, key = parse_bos_path(raw_path) bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip() bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip() bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip() if not bucket or not key or not bos_endpoint or not bos_access_key_id or not bos_secret_access_key: return None target.parent.mkdir(parents=True, exist_ok=True) temp_target = target.with_name(target.name + ".part") with contextlib.suppress(FileNotFoundError): temp_target.unlink() try: client, _ = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key) client.get_object_to_file(bucket, key, str(temp_target)) temp_target.replace(target) except Exception as exc: with contextlib.suppress(FileNotFoundError): temp_target.unlink() raise RuntimeError(f"下载 BOS 输入失败: {default_name}") from exc return target def signed_bos_source_url(raw_path: str) -> str: bucket, key = parse_bos_path(raw_path) if not bucket or not key: return "" bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip() bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip() bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip() if not bos_endpoint or not bos_access_key_id or not bos_secret_access_key: return "" client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key) return text_value( client.generate_pre_signed_url( bucket, key, expiration_in_seconds=3600, protocol=scheme, ) ).strip() def remote_video_url_for_source(source: dict[str, Any]) -> str: preview_url = str(source.get("previewURL", "")).strip() parsed_preview = urlparse(preview_url) if parsed_preview.scheme.lower() in {"http", "https"}: return preview_url raw_path = str(source.get("path", "")).strip() if raw_path.startswith("bos://"): return signed_bos_source_url(raw_path) return "" def is_archive_path(path: pathlib.Path) -> bool: name = path.name.lower() return any(name.endswith(suffix) for suffix in ARCHIVE_FILE_SUFFIXES) def safe_extract_archive(path: pathlib.Path, destination: pathlib.Path) -> list[str]: destination.mkdir(parents=True, exist_ok=True) destination_resolved = destination.resolve() extracted_files: list[str] = [] def ensure_child(member_name: str) -> None: target = (destination / member_name).resolve() if target != destination_resolved and destination_resolved not in target.parents: raise RuntimeError(f"archive entry escapes target directory: {member_name}") if path.name.lower().endswith(".zip"): with zipfile.ZipFile(path) as archive: for member in archive.infolist(): if not member.filename or member.is_dir(): continue ensure_child(member.filename) extracted_files.append(member.filename) archive.extractall(destination) return extracted_files with tarfile.open(path, "r:*") as archive: members = archive.getmembers() for member in members: if not member.name or member.isdir(): continue ensure_child(member.name) extracted_files.append(member.name) archive.extractall(destination) return extracted_files def artifact_slug(value: str) -> str: normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-") return normalized or "artifact" def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str: suffix = path.suffix.lower() if suffix == ".json": return "json" if suffix in VIDEO_FILE_SUFFIXES: return "video" if suffix in IMAGE_FILE_SUFFIXES: return "image" return fallback def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None: cleaned = {key: json_compatible(value) for key, value in metadata.items() if value not in ("", None, [], {})} return cleaned or None def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None: if path.suffix.lower() != ".json": return None for attempt in range(PAYLOAD_FILE_READ_ATTEMPTS): if not path.exists(): if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS: time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS) continue return None try: with path.open("r", encoding="utf-8") as handle: value = json.load(handle) except (FileNotFoundError, OSError, json.JSONDecodeError): if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS: time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS) continue return None except Exception: return None if isinstance(value, dict): return value if isinstance(value, list): return {"items": value, "itemCount": len(value)} return {"value": value} return None def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None: outputs = node_data.get("outputs", []) or [] if name: for output in outputs: if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name: return output if 0 <= index < len(outputs): return outputs[index] return None def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]: with path.open("rb") as handle: raw = handle.read(PREVIEW_TEXT_LIMIT + 1) truncated = len(raw) > PREVIEW_TEXT_LIMIT text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace") lines = text.splitlines() if len(lines) > PREVIEW_TEXT_LINE_LIMIT: text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT]) truncated = True line_count = max(1, text.count("\n") + 1) if text else 0 return text, truncated, line_count def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]: text, truncated, line_count = read_text_sample(path) return { "preview": { "kind": "text", "format": format_name, "text": text, "truncated": truncated, "lineCount": line_count, } } def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]: columns: list[str] = [] rows: list[dict[str, Any]] = [] total_rows = 0 with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: reader = csv.reader(handle, delimiter=delimiter) for index, row in enumerate(reader): if index == 0: columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)] continue total_rows += 1 if len(rows) >= PREVIEW_TABLE_ROW_LIMIT: continue width = max(len(columns), len(row)) row_payload: dict[str, Any] = {} for column_index in range(width): column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}" row_payload[column_name] = row[column_index] if column_index < len(row) else "" rows.append(row_payload) return { "preview": { "kind": "table", "format": format_name, "columns": columns, "rows": rows, "rowCount": total_rows, "truncated": total_rows > len(rows), } } def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]: columns: list[str] = [] rows: list[dict[str, Any]] = [] with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: reader = csv.reader(handle, delimiter=delimiter) for index, row in enumerate(reader): if index == 0: columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)] continue width = max(len(columns), len(row)) row_payload: dict[str, Any] = {} for column_index in range(width): column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}" row_payload[column_name] = row[column_index] if column_index < len(row) else "" rows.append(row_payload) return columns, rows def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]: rows: list[Any] = [] line_count = 0 with path.open("r", encoding="utf-8", errors="replace") as handle: for raw_line in handle: line = raw_line.strip() if not line: continue line_count += 1 if len(rows) >= PREVIEW_TABLE_ROW_LIMIT: continue try: rows.append(json.loads(line)) except Exception: rows.append({"value": line}) columns: list[str] = [] for row in rows: if isinstance(row, dict): for key in row.keys(): if key not in columns: columns.append(key) return { "preview": { "kind": "table", "format": "jsonl", "columns": columns, "rows": rows, "rowCount": line_count, "truncated": line_count > len(rows), } } def inspect_video_file(path: pathlib.Path) -> dict[str, Any]: cv2, _ = require_cv2() capture = cv2.VideoCapture(str(path)) if not capture.isOpened(): return {} try: fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0 payload = clean_metadata( { "frameRate": round(fps, 3) if fps > 0 else None, "frameCount": frame_count or None, "resolution": f"{width}x{height}" if width > 0 and height > 0 else None, "duration": round(duration, 3) if duration > 0 else None, } ) return payload or {} finally: capture.release() def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]: payload: dict[str, Any] = { "preview": { "kind": "image", "format": path.suffix.lower().lstrip(".") or "image", } } size_bytes = path.stat().st_size if path.exists() else 0 if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: payload["preview"]["url"] = data_url_for_path(path, mime_type) return payload def preview_media_payload(path: pathlib.Path, mime_type: str, kind: str) -> dict[str, Any]: preview: dict[str, Any] = { "kind": kind, "format": path.suffix.lower().lstrip(".") or kind, } if kind == "video": preview.update(inspect_video_file(path)) size_bytes = path.stat().st_size if path.exists() else 0 if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: preview["url"] = data_url_for_path(path, mime_type) return {"preview": preview} def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]: try: relative = candidate.relative_to(root).as_posix() except ValueError: relative = candidate.name items.append({"path": relative, "sizeBytes": candidate.stat().st_size}) return items def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]: files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()] extensions: dict[str, int] = {} total_bytes = 0 for candidate in files: extension = candidate.suffix.lower() or "" extensions[extension] = extensions.get(extension, 0) + 1 total_bytes += candidate.stat().st_size ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0])) return { "root": str(root), "fileCount": len(files), "totalBytes": total_bytes, "topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]], "files": sample_file_entries(files, root), "truncated": len(files) > PREVIEW_FILE_LIST_LIMIT, } def optional_json_file(path: pathlib.Path) -> Any: if not path.exists(): return None try: with path.open("r", encoding="utf-8") as handle: return json.load(handle) except Exception: return None def optional_jsonl_rows(path: pathlib.Path) -> list[Any]: rows: list[Any] = [] if not path.exists(): return rows try: with path.open("r", encoding="utf-8", errors="replace") as handle: for raw_line in handle: line = raw_line.strip() if not line: continue rows.append(json.loads(line)) if len(rows) >= PREVIEW_TABLE_ROW_LIMIT: break except Exception: return [] return rows def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None: candidates = [path] if path.is_dir() else [path.parent] candidates.extend(list((path if path.is_dir() else path.parent).parents)) for candidate in candidates: meta_dir = candidate / "meta" if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")): return candidate return None def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]: tree = summarize_directory_tree(root) meta_dir = root / "meta" info = optional_json_file(meta_dir / "info.json") stats = optional_json_file(meta_dir / "stats.json") tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl") episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl") parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()] video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES] return { "preview": { "kind": "dataset", "format": "lerobot", "root": str(root), "summary": tree, "info": info, "stats": stats, "tasks": tasks, "episodes": episodes, "parquetFiles": sample_file_entries(parquet_files, root), "parquetCount": len(parquet_files), "videoCount": len(video_files), } } def sqlite_table_names(connection: sqlite3.Connection) -> list[str]: rows = connection.execute( "select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name" ).fetchall() return [str(row[0]) for row in rows] def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]: connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro&immutable=1", uri=True) try: tables = sqlite_table_names(connection) table_summaries: list[dict[str, Any]] = [] for table_name in tables[:12]: count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0] table_summaries.append({"name": table_name, "rowCount": int(count)}) payload: dict[str, Any] = { "preview": { "kind": "dataset", "format": "sqlite", "tables": table_summaries, "tableCount": len(tables), } } if "topics" in tables and "messages" in tables: topic_rows = connection.execute( "select id, name, type, serialization_format from topics order by id" ).fetchall() message_counts = { int(topic_id): int(count) for topic_id, count in connection.execute( "select topic_id, count(*) from messages group by topic_id order by topic_id" ).fetchall() } payload["preview"]["format"] = "rosbag2" payload["preview"]["topics"] = [ { "id": int(topic_id), "name": str(name), "type": str(topic_type), "serialization": str(serialization), "messageCount": message_counts.get(int(topic_id), 0), } for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT] ] payload["preview"]["messageCount"] = int(sum(message_counts.values())) return payload finally: connection.close() def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]: suffix = path.suffix.lower() if suffix == ".db3": return summarize_sqlite_payload(path) header = "" with path.open("rb") as handle: header = handle.read(32).decode("utf-8", errors="replace") format_name = "mcap" if suffix == ".mcap" else "rosbag1" return { "preview": { "kind": "dataset", "format": format_name, "header": header.strip(), "note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。", } } def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None: direct = params.get("SOURCE_PATH", "").strip() if direct and pathlib.Path(direct).exists(): return pathlib.Path(direct) for artifact in inputs: payload = artifact.get("data") if isinstance(payload, dict): candidate = payload.get("sourcePath") if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists(): return pathlib.Path(candidate) source_path = artifact.get("path") if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists(): return pathlib.Path(source_path) return None def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None: data = artifact.get("data") if not isinstance(data, dict): return None preview = data.get("preview") if isinstance(preview, dict): return preview return None def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]: extensions: dict[str, int] = {} files: list[dict[str, Any]] = [] for artifact in inputs: name = str(artifact.get("name", "") or artifact.get("path", "")).strip() extension = pathlib.Path(name).suffix.lower() or "" extensions[extension] = extensions.get(extension, 0) + 1 files.append( { "path": str(artifact.get("path", "")).strip() or name, "name": name, "size": str(artifact.get("size", "")), } ) ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0])) return { "fileCount": len(files), "topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]], "files": files[:PREVIEW_FILE_LIST_LIMIT], "truncated": len(files) > PREVIEW_FILE_LIST_LIMIT, } def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str: for artifact in inputs: raw_path = str(artifact.get("path", "")).strip() if not raw_path: continue parts = pathlib.PurePosixPath(raw_path).parts for marker in ("meta", "data", "videos"): if marker in parts: index = parts.index(marker) if index > 0: return str(pathlib.PurePosixPath(*parts[:index])) return "" def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]: info = None stats = None tasks: list[Any] = [] episodes: list[Any] = [] parquet_files: list[dict[str, Any]] = [] video_count = 0 for artifact in inputs: name = str(artifact.get("name", "")).strip() path = str(artifact.get("path", "")).strip() data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} preview = artifact_preview_payload(artifact) or {} if name == "info.json" and isinstance(data, dict): info = data elif name == "stats.json" and isinstance(data, dict): stats = data elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list): tasks = preview["rows"] elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list): episodes = preview["rows"] if name.endswith(".parquet"): parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))}) if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video": video_count += 1 return { "preview": { "kind": "dataset", "format": "lerobot", "root": infer_lerobot_root_from_inputs(inputs), "summary": summarize_input_artifacts(inputs), "info": info, "stats": stats, "tasks": tasks, "episodes": episodes, "parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT], "parquetCount": len(parquet_files), "videoCount": video_count, } } def parse_positive_float_param(params: dict[str, str], key: str, default: float) -> float: raw = params.get(key, "").strip() if not raw: return default try: parsed = float(raw) except Exception as exc: raise RuntimeError(f"{key} must be numeric") from exc if not math.isfinite(parsed): raise RuntimeError(f"{key} must be finite") if parsed <= 0: raise RuntimeError(f"{key} must be greater than 0") return parsed def parse_unit_interval_param(params: dict[str, str], key: str, default: float) -> float: raw = params.get(key, "").strip() if not raw: return default try: parsed = float(raw) except Exception as exc: raise RuntimeError(f"{key} must be numeric") from exc if not math.isfinite(parsed): raise RuntimeError(f"{key} must be finite") if parsed < 0 or parsed > 1: raise RuntimeError(f"{key} must be in the range [0, 1]") return parsed def float_value(value: Any, default: float) -> float: if value is None: return default if isinstance(value, str) and not value.strip(): return default try: return float(value) except Exception: return default def parse_positive_int_param(params: dict[str, str], key: str, default: int) -> int: raw = params.get(key, "").strip() if not raw: return default if not EXACT_POSITIVE_INT_RE.fullmatch(raw): if INTEGERISH_RE.fullmatch(raw): try: parsed_integerish = int(raw) except Exception: parsed_integerish = 0 if parsed_integerish <= 0: raise RuntimeError(f"{key} must be greater than 0") raise RuntimeError(f"{key} must be an exact positive integer") raise RuntimeError(f"{key} must be an integer") parsed = int(raw) if parsed > MAX_SAFE_INTEGER: raise RuntimeError(f"{key} must be a safe integer") return parsed def parse_bool_param(params: dict[str, str], key: str, default: bool = False) -> bool: raw = params.get(key, "").strip().lower() if not raw: return default if raw in TRUE_ENV_VALUES: return True if raw in FALSE_ENV_VALUES: return False raise RuntimeError(f"{key} must be a boolean") def first_payload_with_keys(inputs: list[dict[str, Any]], *keys: str) -> dict[str, Any] | None: for artifact in inputs: payload = load_artifact_payload(artifact) if isinstance(payload, dict) and all(key in payload for key in keys): return payload return None def common_input_root(paths: list[str]) -> str: filtered = [item for item in paths if item] if not filtered: return "" if len(filtered) == 1: return str(pathlib.Path(filtered[0]).parent) try: return os.path.commonpath(filtered) except Exception: return str(pathlib.Path(filtered[0]).parent) def artifact_preview_url(artifact: dict[str, Any]) -> str: preview = artifact_preview_payload(artifact) or {} value = preview.get("url") return value if isinstance(value, str) else "" def source_ref_from_artifact(artifact: dict[str, Any], relative_path: str = "") -> dict[str, Any]: metadata = artifact.get("metadata") if isinstance(artifact.get("metadata"), dict) else {} return clean_metadata( { "name": str(artifact.get("name", "")).strip(), "path": str(artifact.get("path", "")).strip(), "relativePath": relative_path, "previewURL": artifact_preview_url(artifact), "mimeType": str(metadata.get("mimeType", "")).strip(), } ) or {} def materialize_artifact_to_path(artifact: dict[str, Any], destination: pathlib.Path) -> bool: raw_path = str(artifact.get("path", "")).strip() destination.parent.mkdir(parents=True, exist_ok=True) if raw_path and pathlib.Path(raw_path).exists(): shutil.copy2(raw_path, destination) return True preview_url = artifact_preview_url(artifact) decoded = decode_data_url(preview_url) if preview_url else None if decoded is not None: _, payload = decoded destination.write_bytes(payload) return True payload = artifact.get("data") if isinstance(payload, dict) and destination.suffix.lower() == ".json": destination.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") return True preview = artifact_preview_payload(artifact) or {} if preview.get("kind") == "text": destination.write_text(str(preview.get("text", "")), encoding="utf-8") return True return False def upload_relative_path(artifact: dict[str, Any], input_root: str, fallback_name: str) -> str: explicit_relative_path = str(artifact.get("relativePath", "")).strip() if explicit_relative_path: return pathlib.PurePosixPath(explicit_relative_path).as_posix().lstrip("/") payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} payload_relative_path = str(payload.get("relativePath", "")).strip() if payload_relative_path: return pathlib.PurePosixPath(payload_relative_path).as_posix().lstrip("/") raw_path = str(artifact.get("path", "")).strip() relative_path = str(fallback_name or artifact.get("name", "")).strip() or "artifact" if raw_path and input_root: try: relative_path = pathlib.Path(raw_path).relative_to(input_root).as_posix() except Exception: relative_path = pathlib.Path(raw_path).name or relative_path if not relative_path: relative_path = pathlib.Path(fallback_name or "artifact").name or "artifact" return pathlib.PurePosixPath(relative_path).as_posix().lstrip("/") def materialized_upload_source(artifact: dict[str, Any], staging_dir: pathlib.Path, relative_path: str) -> pathlib.Path | None: raw_path = str(artifact.get("path", "")).strip() if raw_path and os.path.exists(raw_path): return pathlib.Path(raw_path) destination = staging_dir / "_materialized_upload_inputs" / relative_path if destination.exists(): return destination if materialize_artifact_to_path(artifact, destination): return destination return None def package_config_payload(item: dict[str, Any], manifest_payload: dict[str, Any]) -> dict[str, Any]: return { "collectionId": item.get("collectionId", ""), "meta": item.get("meta", {}), "segmentDurationSeconds": manifest_payload.get("segmentDurationSeconds"), "sourceVideo": item.get("sourceVideo", ""), "startSecond": item.get("startSecond"), "endSecond": item.get("endSecond"), "duration": item.get("duration") or item.get("clipDuration"), "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, } def package_related_artifact_names(manifest_payload: dict[str, Any]) -> set[str]: names: set[str] = set() for item in manifest_payload.get("items", []): if not isinstance(item, dict): continue clip_id = str(item.get("clipId", "")).strip() clip_name = str(item.get("clipName", "")).strip() annotation_name = str(item.get("annotationFile", "")).strip() or (f"{clip_id}_annotation.json" if clip_id else "") quality_name = f"{clip_id}_quality.json" if clip_id else "" config_name = f"{clip_id}_config.json" if clip_id else "" for candidate in (clip_name, annotation_name, quality_name, config_name): if candidate: names.add(candidate) return names def materialize_package_artifacts_from_manifest( run_id: str, node_id: str, staging_dir: pathlib.Path, manifest_payload: dict[str, Any], node_data: dict[str, Any], ) -> list[dict[str, Any]]: archive_sources = manifest_payload.get("archiveSources", {}) if isinstance(manifest_payload.get("archiveSources"), dict) else {} binding = output_binding(node_data) materialized_artifacts: list[dict[str, Any]] = [] materialized_root = staging_dir / "_materialized_upload_inputs" for item in manifest_payload.get("items", []): if not isinstance(item, dict): continue clip_id = str(item.get("clipId", "")).strip() if not clip_id: continue package_name = str(item.get("packageName", "")).strip() or f"{clip_id}_package" clip_name = str(item.get("clipName", "")).strip() or f"{clip_id}.mp4" package_relative_dir = pathlib.PurePosixPath("packages") / package_name package_dir = materialized_root / package_relative_dir package_dir.mkdir(parents=True, exist_ok=True) clip_destination = package_dir / clip_name direct_clip_path = str(item.get("clipPath", "")).strip() storage_path = str(item.get("storagePath", "")).strip() clip_candidates = [direct_clip_path] if storage_path: clip_candidates.append(str(pathlib.Path(storage_path) / clip_name)) clip_source = next( ( pathlib.Path(candidate) for candidate in clip_candidates if candidate and pathlib.Path(candidate).is_file() ), None, ) if clip_source is None: clip_source = materialize_clip_from_source(item, staging_dir, archive_sources) if clip_source is not None and clip_source.exists(): clip_size = clip_source.stat().st_size clip_mime_type = mimetypes.guess_type(clip_name)[0] or "video/mp4" clip_artifact = { "id": f"{run_id}-{node_id}-{package_name}-{clip_name}", "runId": run_id, "nodeId": node_id, "name": clip_name, "type": "video", "size": format_size(clip_size), "path": str(clip_source), "metadata": {"mimeType": clip_mime_type}, "data": { "relativePath": (package_relative_dir / clip_name).as_posix(), "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, }, } clip_artifact["relativePath"] = (package_relative_dir / clip_name).as_posix() materialized_artifacts.append(clip_artifact) annotation_name = str(item.get("annotationFile", "")).strip() or f"{clip_id}_annotation.json" annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {} annotation_artifact = write_json_artifact(run_id, node_id, package_dir, annotation_name, annotation_payload, output=binding) annotation_artifact["relativePath"] = (package_relative_dir / annotation_name).as_posix() materialized_artifacts.append(annotation_artifact) quality_name = f"{clip_id}_quality.json" quality_payload = { "clip_id": clip_id, "source_video": item.get("sourceVideo", ""), **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}), } quality_artifact = write_json_artifact(run_id, node_id, package_dir, quality_name, quality_payload, output=binding) quality_artifact["relativePath"] = (package_relative_dir / quality_name).as_posix() materialized_artifacts.append(quality_artifact) config_name = f"{clip_id}_config.json" config_artifact = write_json_artifact( run_id, node_id, package_dir, config_name, package_config_payload(item, manifest_payload), output=binding, ) config_artifact["relativePath"] = (package_relative_dir / config_name).as_posix() materialized_artifacts.append(config_artifact) return materialized_artifacts def materialize_input_tree( inputs: list[dict[str, Any]], output_dir: pathlib.Path, ) -> tuple[pathlib.Path | None, dict[str, dict[str, Any]]]: paths = [str(item.get("path", "")).strip() for item in inputs if str(item.get("path", "")).strip()] root = common_input_root(paths) if not root: return None, {} target_root = output_dir / "_materialized_inputs" if target_root.exists(): shutil.rmtree(target_root) target_root.mkdir(parents=True, exist_ok=True) source_refs: dict[str, dict[str, Any]] = {} for index, artifact in enumerate(inputs): raw_path = str(artifact.get("path", "")).strip() relative_path = str(artifact.get("name", "")).strip() or f"artifact-{index}" if raw_path: try: relative_path = pathlib.Path(raw_path).relative_to(root).as_posix() except Exception: relative_path = pathlib.Path(raw_path).name or relative_path destination = target_root / relative_path if materialize_artifact_to_path(artifact, destination): source_refs[relative_path] = source_ref_from_artifact(artifact, relative_path) if destination.is_file() and is_archive_path(destination): extracted_files = safe_extract_archive(destination, target_root) archive_preview_url = artifact_preview_url(artifact) archive_key = artifact_slug(hashlib.sha256(archive_preview_url.encode("utf-8")).hexdigest()[:16]) if archive_preview_url else artifact_slug(relative_path) for extracted_relative_path in extracted_files: normalized_relative_path = pathlib.PurePosixPath(extracted_relative_path).as_posix().lstrip("/") source_refs[normalized_relative_path] = clean_metadata( { "name": pathlib.PurePosixPath(normalized_relative_path).name, "path": str(target_root / normalized_relative_path), "relativePath": normalized_relative_path, "archiveKey": archive_key, "archivePreviewURL": archive_preview_url, "archiveMemberPath": normalized_relative_path, "archiveName": destination.name, "mimeType": mimetypes.guess_type(normalized_relative_path)[0] or "", } ) or {} return target_root, source_refs def detect_xplatform_collections(root: pathlib.Path) -> list[pathlib.Path]: if not root.exists(): return [] candidates: list[pathlib.Path] = [] if (root / "meta.json").exists(): return [root] for child in sorted(root.iterdir()): if not child.is_dir(): continue if (child / "meta.json").exists(): candidates.append(child) continue if any(candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES for candidate in child.rglob("*")): candidates.append(child) return candidates def normalize_collection_relative_path(value: pathlib.PurePath | str) -> str: text = value.as_posix() if isinstance(value, pathlib.PurePath) else str(value or "") normalized = text.strip().strip("/") if normalized in {"", "."}: return "" return normalized def collection_name_from_source_path(source_path: str, fallback_name: str = "collection") -> str: candidate = pathlib.PurePosixPath(str(source_path or "").rstrip("/")).name return candidate or fallback_name def parse_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]: rows: dict[str, dict[str, Any]] = {} if not isinstance(value, list): return rows for item in value: if not isinstance(item, dict): continue filename = str(item.get("filename", "")).strip() if not filename: continue rows[filename] = item return rows def collection_file_entry( root: pathlib.Path, collection_dir: pathlib.Path, candidate: pathlib.Path, collection_id: str, meta: dict[str, Any], video_meta_rows: dict[str, dict[str, Any]], ) -> dict[str, Any]: relative_path = candidate.relative_to(root).as_posix() collection_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root)) collection_relative_file_path = candidate.relative_to(collection_dir).as_posix() mime_type = mimetypes.guess_type(candidate.name)[0] or "" entry: dict[str, Any] = { "collectionId": collection_id, "collectionName": collection_dir.name, "collectionPath": str(collection_dir), "collectionRelativePath": collection_relative_path, "meta": meta, "relativePath": relative_path, "path": str(candidate), "sizeBytes": candidate.stat().st_size, "mimeType": mime_type, "source": {"path": str(candidate), "relativePath": relative_path, "mimeType": mime_type}, } if candidate.suffix.lower() == ".json": payload = optional_json_file(candidate) if payload is not None: entry["data"] = payload if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES: matched_meta = video_meta_rows.get(candidate.name, {}) entry.update( { "videoFolder": collection_relative_file_path.split("/", 1)[0] if "/" in collection_relative_file_path else "", "filename": candidate.name, "codec": matched_meta.get("codec"), "resolution": matched_meta.get("resolution"), "frameRate": matched_meta.get("frameRate"), "duration": matched_meta.get("duration"), "startTime": matched_meta.get("startTime"), } ) return entry def collection_summary( root: pathlib.Path, collection_dir: pathlib.Path, meta: dict[str, Any], video_meta_rows: dict[str, dict[str, Any]], ) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: files = [candidate for candidate in sorted(collection_dir.rglob("*")) if candidate.is_file()] videos = [candidate for candidate in files if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES] total_bytes = sum(candidate.stat().st_size for candidate in files) required = ["meta.json", "video_meta.json"] missing_required = [] if not (collection_dir / "meta.json").exists(): missing_required.append("meta.json") if not any((collection_dir / candidate).exists() for candidate in XPLATFORM_VIDEO_META_FILES): missing_required.append("video_meta.json") has_intrinsics = (collection_dir / "intrinsics.json").exists() collection_root_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root)) collection_id = collection_key(collection_root_relative_path, collection_dir.name) items: list[dict[str, Any]] = [] entries: list[dict[str, Any]] = [] for candidate in files: entry = collection_file_entry(root, collection_dir, candidate, collection_id, meta, video_meta_rows) entries.append(entry) if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES: items.append(dict(entry)) summary = { "id": collection_id, "name": collection_dir.name, "path": str(collection_dir), "relativePath": collection_root_relative_path, "meta": meta, "hasIntrinsics": has_intrinsics, "hasVideoMeta": bool(video_meta_rows), "requiredFiles": required, "missingRequiredFiles": missing_required, "videoCount": len(videos), "fileCount": len(files), "totalBytes": total_bytes, "valid": len(missing_required) == 0 and len(videos) > 0, } return summary, items, entries def source_path_join(root: str, relative_path: str) -> str: if not root: return relative_path return root.rstrip("/") + "/" + relative_path.lstrip("/") def manifest_source_ref(entry: dict[str, Any], relative_path: str, source_path: str) -> dict[str, Any]: source = entry.get("source") if isinstance(entry.get("source"), dict) else {} storage_path = str(source.get("path", "") or entry.get("storagePath", "") or source_path_join(source_path, relative_path)).strip() return clean_metadata( { "path": storage_path, "relativePath": relative_path, "previewURL": str(source.get("previewURL", "")).strip(), "mimeType": str(entry.get("mimeType", "")).strip(), } ) or {} def manifest_entry_json_payload(entry: Any, source_path: str = "") -> Any: if not isinstance(entry, dict): return None inline_data = entry.get("data") if inline_data is not None: return inline_data source = entry.get("source") if isinstance(entry.get("source"), dict) else {} candidates = [str(source.get("path", "")).strip(), str(entry.get("storagePath", "")).strip(), str(entry.get("path", "")).strip()] for raw_candidate in candidates: if not raw_candidate: continue if "://" in raw_candidate and not raw_candidate.startswith("file://"): continue candidate_text = raw_candidate[len("file://") :] if raw_candidate.startswith("file://") else raw_candidate candidate_path = pathlib.Path(candidate_text) if not candidate_path.is_absolute(): normalized_source_path = str(source_path or "").strip() if normalized_source_path and "://" not in normalized_source_path: candidate_path = pathlib.Path(normalized_source_path) / candidate_text if candidate_path.is_file(): return optional_json_file(candidate_path) return None def manifest_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]: if isinstance(value, dict) and isinstance(value.get("items"), list): return parse_video_meta_rows(value.get("items")) return parse_video_meta_rows(value) def collection_key(relative_path: str, fallback_name: str = "collection") -> str: normalized = relative_path.strip().strip("/") if not normalized: return artifact_slug(fallback_name) return artifact_slug(normalized) def fallback_manifest_collections(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]: grouped: dict[str, list[dict[str, Any]]] = {} for entry in entries: relative_path = str(entry.get("path", "")).strip().lstrip("/") if not relative_path: continue parts = pathlib.PurePosixPath(relative_path).parts collection_relative_path = parts[0] if len(parts) >= 2 else "" grouped.setdefault(collection_relative_path, []).append({**entry, "path": relative_path}) collection_summaries: list[dict[str, Any]] = [] items: list[dict[str, Any]] = [] indexed_entries: list[dict[str, Any]] = [] for collection_relative_path in sorted(grouped.keys()): collection_entries = grouped[collection_relative_path] collection_name = ( pathlib.PurePosixPath(collection_relative_path).name if collection_relative_path else collection_name_from_source_path(source_path) ) collection_id = collection_key(collection_relative_path, collection_name) meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json" intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json" meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None) intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None) video_meta_entry = next( ( entry for entry in collection_entries if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name) ), None, ) meta_payload = manifest_entry_json_payload(meta_entry, source_path) video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path) meta = meta_payload if isinstance(meta_payload, dict) else {} video_meta_rows = manifest_video_meta_rows(video_meta_payload) videos = [ entry for entry in collection_entries if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES ] total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries) required = ["meta.json", "video_meta.json"] missing_required = [] if meta_entry is None: missing_required.append("meta.json") if video_meta_entry is None: missing_required.append("video_meta.json") collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path) collection_summaries.append( { "id": collection_id, "name": collection_name, "path": collection_path, "relativePath": collection_relative_path, "meta": meta, "hasIntrinsics": intrinsics_entry is not None, "hasVideoMeta": bool(video_meta_rows), "requiredFiles": required, "missingRequiredFiles": missing_required, "videoCount": len(videos), "fileCount": len(collection_entries), "totalBytes": total_bytes, "valid": len(missing_required) == 0 and len(videos) > 0, } ) for collection_entry in collection_entries: relative_path = str(collection_entry.get("path", "")).strip() entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or "" indexed_entry = { "collectionId": collection_id, "collectionName": collection_name, "collectionPath": collection_path, "collectionRelativePath": collection_relative_path, "meta": meta, "relativePath": relative_path, "path": source_path_join(source_path, relative_path), "sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0), "mimeType": entry_mime_type, "source": manifest_source_ref(collection_entry, relative_path, source_path), } if "data" in collection_entry: indexed_entry["data"] = normalize_json_payload(collection_entry.get("data")) if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES: collection_relative_video_path = ( str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) if collection_relative_path else relative_path ) matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) indexed_entry.update( { "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", "filename": pathlib.PurePosixPath(relative_path).name, "codec": matched_meta.get("codec"), "resolution": matched_meta.get("resolution"), "frameRate": matched_meta.get("frameRate"), "duration": matched_meta.get("duration"), "startTime": matched_meta.get("startTime"), } ) indexed_entries.append(indexed_entry) for video_entry in videos: relative_path = str(video_entry.get("path", "")).strip() collection_relative_video_path = ( str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) if collection_relative_path else relative_path ) matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) items.append( { "collectionId": collection_id, "collectionName": collection_name, "collectionPath": collection_path, "collectionRelativePath": collection_relative_path, "meta": meta, "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", "filename": pathlib.PurePosixPath(relative_path).name, "relativePath": relative_path, "path": source_path_join(source_path, relative_path), "sizeBytes": int(video_entry.get("sizeBytes", 0) or 0), "codec": matched_meta.get("codec"), "resolution": matched_meta.get("resolution"), "frameRate": matched_meta.get("frameRate"), "duration": matched_meta.get("duration"), "startTime": matched_meta.get("startTime"), "source": manifest_source_ref(video_entry, relative_path, source_path), } ) return { "datasetType": "x-platform", "rootPath": source_path, "collectionCount": len(collection_summaries), "videoCount": len(items), "collections": collection_summaries, "items": items, "entries": indexed_entries, } def scan_xplatform_manifest_entries(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]: normalized_entries: list[dict[str, Any]] = [] meta_roots: set[str] = set() auxiliary_roots: set[str] = set() has_root_metadata = False for entry in entries: relative_path = str(entry.get("path", "")).strip().lstrip("/") if not relative_path: continue normalized_entry = {**entry, "path": relative_path} normalized_entries.append(normalized_entry) path_obj = pathlib.PurePosixPath(relative_path) parent = path_obj.parent.as_posix() if parent in {".", ""}: parent = "" file_name = path_obj.name.lower() if file_name == "meta.json": if parent: meta_roots.add(parent) else: has_root_metadata = True elif is_xplatform_video_meta_name(file_name) or file_name == "intrinsics.json": if parent: auxiliary_roots.add(parent) else: has_root_metadata = True root_candidates = meta_roots or auxiliary_roots if has_root_metadata: root_candidates = set(root_candidates) root_candidates.add("") if not root_candidates: return fallback_manifest_collections(normalized_entries, source_path) ordered_roots = sorted(root_candidates, key=lambda value: (-len(pathlib.PurePosixPath(value).parts), value)) grouped: dict[str, list[dict[str, Any]]] = {root: [] for root in ordered_roots} unmatched_entries: list[dict[str, Any]] = [] for entry in normalized_entries: relative_path = str(entry.get("path", "")).strip() matched_root = next( ( root for root in ordered_roots if ( (not root) or relative_path == root or relative_path.startswith(root + "/") ) ), None, ) if matched_root is not None: grouped.setdefault(matched_root, []).append(entry) else: unmatched_entries.append(entry) collection_summaries: list[dict[str, Any]] = [] items: list[dict[str, Any]] = [] indexed_entries: list[dict[str, Any]] = [] for collection_relative_path in sorted(grouped.keys()): collection_entries = grouped[collection_relative_path] if not collection_entries: continue collection_name = ( pathlib.PurePosixPath(collection_relative_path).name if collection_relative_path else collection_name_from_source_path(source_path) ) collection_id = collection_key(collection_relative_path, collection_name) meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json" intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json" meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None) intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None) video_meta_entry = next( ( entry for entry in collection_entries if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name) ), None, ) meta_payload = manifest_entry_json_payload(meta_entry, source_path) video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path) meta = meta_payload if isinstance(meta_payload, dict) else {} video_meta_rows = manifest_video_meta_rows(video_meta_payload) videos = [ entry for entry in collection_entries if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES ] total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries) required = ["meta.json", "video_meta.json"] missing_required = [] if meta_entry is None: missing_required.append("meta.json") if video_meta_entry is None: missing_required.append("video_meta.json") collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path) collection_summaries.append( { "id": collection_id, "name": collection_name, "path": collection_path, "relativePath": collection_relative_path, "meta": meta, "hasIntrinsics": intrinsics_entry is not None, "hasVideoMeta": bool(video_meta_rows), "requiredFiles": required, "missingRequiredFiles": missing_required, "videoCount": len(videos), "fileCount": len(collection_entries), "totalBytes": total_bytes, "valid": len(missing_required) == 0 and len(videos) > 0, } ) for collection_entry in collection_entries: relative_path = str(collection_entry.get("path", "")).strip() entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or "" indexed_entry = { "collectionId": collection_id, "collectionName": collection_name, "collectionPath": collection_path, "collectionRelativePath": collection_relative_path, "meta": meta, "relativePath": relative_path, "path": source_path_join(source_path, relative_path), "sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0), "mimeType": entry_mime_type, "source": manifest_source_ref(collection_entry, relative_path, source_path), } if "data" in collection_entry: indexed_entry["data"] = normalize_json_payload(collection_entry.get("data")) if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES: collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) indexed_entry.update( { "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", "filename": pathlib.PurePosixPath(relative_path).name, "codec": matched_meta.get("codec"), "resolution": matched_meta.get("resolution"), "frameRate": matched_meta.get("frameRate"), "duration": matched_meta.get("duration"), "startTime": matched_meta.get("startTime"), } ) indexed_entries.append(indexed_entry) for video_entry in videos: relative_path = str(video_entry.get("path", "")).strip() collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path)) matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {}) items.append( { "collectionId": collection_id, "collectionName": collection_name, "collectionPath": collection_path, "collectionRelativePath": collection_relative_path, "meta": meta, "videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "", "filename": pathlib.PurePosixPath(relative_path).name, "relativePath": relative_path, "path": source_path_join(source_path, relative_path), "sizeBytes": int(video_entry.get("sizeBytes", 0) or 0), "codec": matched_meta.get("codec"), "resolution": matched_meta.get("resolution"), "frameRate": matched_meta.get("frameRate"), "duration": matched_meta.get("duration"), "startTime": matched_meta.get("startTime"), "source": manifest_source_ref(video_entry, relative_path, source_path), } ) if unmatched_entries: fallback_payload = fallback_manifest_collections(unmatched_entries, source_path) collection_summaries.extend(fallback_payload.get("collections", [])) items.extend(fallback_payload.get("items", [])) indexed_entries.extend(fallback_payload.get("entries", [])) return { "datasetType": "x-platform", "rootPath": source_path, "collectionCount": len(collection_summaries), "videoCount": len(items), "collections": collection_summaries, "items": items, "entries": indexed_entries, } def with_xplatform_source_context(payload: dict[str, Any], source_manifest: dict[str, Any]) -> dict[str, Any]: for key in ( "dataSource", "sourcePath", "scanMode", "listedCount", "newCount", "sourceFingerprint", "bucket", "prefix", "pattern", "endpoint", ): if key in source_manifest: payload[key] = source_manifest.get(key) return payload def attach_xplatform_catalog_payload(payload: dict[str, Any], catalog_payload: dict[str, Any]) -> dict[str, Any]: payload["catalogCollectionCount"] = int(catalog_payload.get("collectionCount", 0) or 0) payload["catalogVideoCount"] = int(catalog_payload.get("videoCount", 0) or 0) payload["catalogCollections"] = catalog_payload.get("collections", []) payload["catalogItems"] = catalog_payload.get("items", []) payload["catalogEntries"] = catalog_payload.get("entries", []) return payload def apply_source_refs_to_items( items: list[dict[str, Any]], source_refs: dict[str, dict[str, Any]], ) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: if not source_refs: return items, {} archive_sources: dict[str, dict[str, Any]] = {} for item in items: relative_path = str(item.get("relativePath", "")).strip() if not relative_path: continue source_ref = source_refs.get(relative_path) if source_ref: archive_key = str(source_ref.get("archiveKey", "")).strip() archive_preview_url = str(source_ref.get("archivePreviewURL", "")).strip() if archive_key and archive_preview_url and archive_key not in archive_sources: archive_sources[archive_key] = clean_metadata( { "name": str(source_ref.get("archiveName", "")).strip() or "archive", "previewURL": archive_preview_url, } ) or {} item_source = {**source_ref, "path": item.get("path", "")} item_source.pop("archivePreviewURL", None) item_source.pop("archiveName", None) item["source"] = item_source return items, archive_sources def scan_xplatform_dataset(root: pathlib.Path, source_refs: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]: collections = detect_xplatform_collections(root) collection_summaries: list[dict[str, Any]] = [] items: list[dict[str, Any]] = [] entries: list[dict[str, Any]] = [] for collection_dir in collections: meta = optional_json_file(collection_dir / "meta.json") meta = meta if isinstance(meta, dict) else {} video_meta_rows, _ = load_collection_video_meta(collection_dir) summary, collection_items, collection_entries = collection_summary(root, collection_dir, meta, video_meta_rows) collection_summaries.append(summary) items.extend(collection_items) entries.extend(collection_entries) items, archive_sources = apply_source_refs_to_items(items, source_refs or {}) entries, entry_archive_sources = apply_source_refs_to_items(entries, source_refs or {}) archive_sources.update(entry_archive_sources) payload = { "datasetType": "x-platform", "rootPath": str(root), "collectionCount": len(collection_summaries), "videoCount": len(items), "collections": collection_summaries, "items": items, "entries": entries, } if archive_sources: payload["archiveSources"] = archive_sources return payload def load_xplatform_index_payload( params: dict[str, str], inputs: list[dict[str, Any]], output_dir: pathlib.Path, ) -> dict[str, Any]: source_manifest = first_payload_with_keys(inputs, "entries") if source_manifest is not None: entries = source_manifest.get("entries") if isinstance(entries, list): manifest_source_path = str(source_manifest.get("sourcePath", "")).strip() payload = scan_xplatform_manifest_entries(entries, manifest_source_path) catalog_entries = source_manifest.get("catalogEntries") if isinstance(catalog_entries, list): catalog_payload = scan_xplatform_manifest_entries(catalog_entries, manifest_source_path) attach_xplatform_catalog_payload(payload, catalog_payload) return with_xplatform_source_context(payload, source_manifest) manifest_source_path = str(source_manifest.get("sourcePath", "")).strip() if manifest_source_path and pathlib.Path(manifest_source_path).exists(): root = pathlib.Path(manifest_source_path) return scan_xplatform_dataset(root if root.is_dir() else root.parent) for artifact in inputs: name = str(artifact.get("name", "")).strip().lower() data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} if name != "source-manifest.json" and not any(key in data for key in ("entriesCount", "catalogEntriesCount", "sourcePath")): continue raise RuntimeError("X 平台数据集索引未能读取 source-manifest 的 entries,拒绝回退到空目录扫描") source_path = resolve_source_path(params, inputs) if source_path is not None: root = source_path if source_path.is_dir() else source_path.parent return scan_xplatform_dataset(root) materialized_root, source_refs = materialize_input_tree(inputs, output_dir) if materialized_root is not None: return scan_xplatform_dataset(materialized_root, source_refs) raise RuntimeError("X 平台数据集索引需要 SOURCE_PATH 或上游输入提供真实数据目录") def normalize_date_value(value: Any) -> str: text = str(value or "").strip() digits = "".join(character for character in text if character.isdigit()) if len(digits) >= 8: return digits[:8] return "" def date_in_range(value: Any, start: str, end: str) -> bool: normalized = normalize_date_value(value) if not normalized: return False if start and normalized < start: return False if end and normalized > end: return False return True def materialize_archive_member( source: dict[str, Any], archive_sources: dict[str, Any], default_name: str, output_dir: pathlib.Path, ) -> pathlib.Path | None: archive_key = str(source.get("archiveKey", "")).strip() archive_source = archive_sources.get(archive_key) if isinstance(archive_sources, dict) else {} archive_source = archive_source if isinstance(archive_source, dict) else {} preview_url = str(archive_source.get("previewURL", "")).strip() member_path = str(source.get("archiveMemberPath", "")).strip() decoded = decode_data_url(preview_url) if preview_url and member_path else None if decoded is None: return None _, payload = decoded archive_name = pathlib.PurePosixPath(str(archive_source.get("name", "")).strip() or "archive").name archive_path = output_dir / "_materialized_sources" / archive_name archive_path.parent.mkdir(parents=True, exist_ok=True) archive_path.write_bytes(payload) destination = output_dir / "_materialized_sources" / default_name destination.parent.mkdir(parents=True, exist_ok=True) if archive_name.lower().endswith(".zip"): with zipfile.ZipFile(archive_path) as archive: with archive.open(member_path) as handle: destination.write_bytes(handle.read()) return destination with tarfile.open(archive_path, "r:*") as archive: extracted = archive.extractfile(member_path) if extracted is None: return None with extracted: destination.write_bytes(extracted.read()) return destination def source_path_from_item( item: dict[str, Any], default_name: str, output_dir: pathlib.Path, archive_sources: dict[str, Any] | None = None, ) -> pathlib.Path: source = item.get("source") if isinstance(item.get("source"), dict) else {} direct = str(source.get("path", "") or item.get("path", "")).strip() if direct and pathlib.Path(direct).exists(): return pathlib.Path(direct) if direct.startswith("bos://"): cached_target = workspace_cached_bos_source_path(direct, output_dir) if cached_target is not None and cached_target.exists() and cached_target.stat().st_size > 0: return cached_target cached_bos_source = download_bos_source(direct, default_name, output_dir) if cached_bos_source is not None: return cached_bos_source preview_url = str(source.get("previewURL", "")).strip() if cached_target is not None and preview_url: cached_remote_source = download_remote_source_to_path(preview_url, cached_target) if cached_remote_source is not None: return cached_remote_source archive_member = materialize_archive_member(source, archive_sources or {}, default_name, output_dir) if archive_member is not None: return archive_member preview_url = str(source.get("previewURL", "")).strip() remote_path = download_remote_source(preview_url, default_name, output_dir) if preview_url else None if remote_path is not None: return remote_path decoded = decode_data_url(preview_url) if preview_url else None if decoded is None: raise RuntimeError(f"无法获取真实输入文件: {default_name}") _, payload = decoded path = output_dir / "_materialized_sources" / default_name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(payload) return path def clip_segment_bounds(item: dict[str, Any]) -> tuple[float, float]: start_second = float(item.get("startSecond") or 0.0) end_second = float(item.get("endSecond") or 0.0) duration = float(item.get("duration") or 0.0) if end_second <= start_second and duration > 0: end_second = start_second + duration if end_second <= start_second: end_second = start_second + 0.1 return start_second, end_second def materialize_clip_from_source( item: dict[str, Any], output_dir: pathlib.Path, archive_sources: dict[str, Any] | None = None, ) -> pathlib.Path | None: source = item.get("source") if isinstance(item.get("source"), dict) else {} source_name = ( str(item.get("sourceVideo", "")).strip() or pathlib.PurePosixPath(str(source.get("path", "")).strip()).name or "source.mp4" ) clip_name = str(item.get("clipName", "")).strip() or f"{pathlib.Path(source_name).stem}_clip.mp4" destination = output_dir / "_materialized_clips" / clip_name if destination.exists(): return destination start_second, end_second = clip_segment_bounds(item) remote_source_url = remote_video_url_for_source(source) if source else "" if remote_source_url: try: write_video_segment_from_remote_url(remote_source_url, destination, start_second, end_second) return destination except RuntimeError as exc: log(f"[operator-runner][warn] fallback to temporary source materialization for {clip_name}: {exc}") if not source: return None source_path = source_path_from_item({"source": source}, source_name, output_dir, archive_sources) write_video_segment(source_path, destination, start_second, end_second) return destination def analyze_video_quality_metrics(path: pathlib.Path, clarity_scale: float) -> dict[str, Any]: cv2, np = require_cv2() capture = cv2.VideoCapture(str(path)) if not capture.isOpened(): raise RuntimeError(f"无法打开视频文件: {path}") try: fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) positions = sampled_frame_positions(frame_count) if not positions: positions = [0] clarities: list[float] = [] shifts: list[tuple[float, float]] = [] skin_ratios: list[float] = [] previous_gray = None previous_position: int | None = None analysis_diagonal = math.sqrt((width or 1) ** 2 + (height or 1) ** 2) for position in positions: if frame_count > 0: capture.set(cv2.CAP_PROP_POS_FRAMES, position) ok, frame = capture.read() if not ok: continue analysis_frame = resize_quality_frame(frame) analysis_height, analysis_width = analysis_frame.shape[:2] analysis_diagonal = math.sqrt((analysis_width or 1) ** 2 + (analysis_height or 1) ** 2) gray = cv2.cvtColor(analysis_frame, cv2.COLOR_BGR2GRAY) laplacian = cv2.Laplacian(gray, cv2.CV_64F).var() clarities.append(float(laplacian)) if previous_gray is not None: gap = max(position - previous_position, 1) if previous_position is not None else 1 shift, _ = cv2.phaseCorrelate(np.float32(previous_gray), np.float32(gray)) shifts.append((float(shift[0]) / gap, float(shift[1]) / gap)) mask = build_skin_mask(analysis_frame) skin_ratios.append(largest_skin_component_ratio(mask)) previous_gray = gray previous_position = position clarity_raw = sum(clarities) / max(len(clarities), 1) clarity = 1.0 - math.exp(-max(clarity_raw, 0.0) / clarity_scale) shake = motion_jitter_ratio(shifts, analysis_diagonal) if shifts else 0.0 hand_occlusion = sum(skin_ratios) / max(len(skin_ratios), 1) if skin_ratios else 0.0 duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0 quality_score = max(0.0, min(1.0, (clarity + (1.0 - min(shake, 1.0)) + (1.0 - min(hand_occlusion, 1.0))) / 3.0)) return { "codec": "opencv", "resolution": f"{width}x{height}" if width > 0 and height > 0 else "", "frameRate": round(fps, 3) if fps > 0 else None, "duration": round(duration, 3) if duration > 0 else None, "clarity": round(clarity, 4), "shake": round(shake, 4), "handOcclusion": round(hand_occlusion, 4), "qualityScore": round(quality_score, 4), "sampleCount": len(clarities), } finally: capture.release() def stable_sample_hit(value: str, percent: float) -> bool: if percent >= 100: return True if percent <= 0: return False digest = hashlib.sha256(value.encode("utf-8")).hexdigest() score = int(digest[:8], 16) / 0xFFFFFFFF return score <= (percent / 100.0) def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None: suffix = path.suffix.lower() if suffix in TABULAR_FILE_SUFFIXES: return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip(".")) if suffix == ".jsonl": return preview_jsonl_payload(path) if suffix in TEXT_FILE_SUFFIXES: format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".") return preview_text_payload(path, format_name) if suffix in IMAGE_FILE_SUFFIXES: return preview_image_payload(path, mime_type) if suffix in ROSBAG_FILE_SUFFIXES: return summarize_rosbag_file(path) if suffix == ".pdf": return {"preview": {"kind": "document", "format": "pdf"}} if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES: return preview_media_payload(path, mime_type or "video/mp4", "video") if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"): return preview_media_payload(path, mime_type or "audio/mpeg", "audio") size_bytes = path.stat().st_size if path.exists() else 0 if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type: return {"preview": {"kind": "binary", "format": suffix.lstrip(".") or "file", "url": data_url_for_path(path, mime_type)}} return None def build_artifact( run_id: str, node_id: str, path: pathlib.Path, artifact_type: str, data: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None, output: dict[str, Any] | None = None, ) -> dict[str, Any]: mime_type, _ = mimetypes.guess_type(path.name) size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8")) sha256_value = "" sha256_metadata: dict[str, Any] = {} if path.exists(): sha256_value, sha256_metadata = artifact_sha256(path, size_bytes) artifact_metadata = clean_metadata( { **(metadata or {}), **sha256_metadata, "sha256": sha256_value, "mimeType": mime_type or "", } ) artifact = { "id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}", "runId": run_id, "nodeId": node_id, "name": path.name, "type": artifact_type, "size": format_size(size_bytes), "path": str(path), } if output: if str(output.get("id", "")).strip(): artifact["portId"] = str(output.get("id", "")).strip() if str(output.get("name", "")).strip(): artifact["portName"] = str(output.get("name", "")).strip() if artifact_metadata: artifact["metadata"] = artifact_metadata if data: artifact["data"] = data return artifact def build_virtual_artifact( run_id: str, node_id: str, name: str, path: str, artifact_type: str, size_bytes: int, data: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None, output: dict[str, Any] | None = None, ) -> dict[str, Any]: artifact = { "id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else name)}-{artifact_slug(name)}", "runId": run_id, "nodeId": node_id, "name": name, "type": artifact_type, "size": format_size(max(0, size_bytes)), "path": path, } if output: if str(output.get("id", "")).strip(): artifact["portId"] = str(output.get("id", "")).strip() if str(output.get("name", "")).strip(): artifact["portName"] = str(output.get("name", "")).strip() if metadata: cleaned = clean_metadata(metadata) if cleaned: artifact["metadata"] = cleaned if data: artifact["data"] = data return artifact def dataset_partition_keys(payload: Any) -> list[str]: if not isinstance(payload, dict): return [] keys: list[str] = [] for key in DATASET_PARTITION_KEYS: if isinstance(payload.get(key), list): keys.append(key) return keys def should_externalize_structured_payload(payload: Any) -> bool: if not isinstance(payload, dict): return False if dataset_partition_keys(payload): return True dataset_type = str(payload.get("datasetType", "")).strip() if dataset_type: return True return False def dataset_payload_summary(payload: Any) -> dict[str, Any]: payload = normalize_json_payload(payload) summary: dict[str, Any] = {} for key in ( "datasetType", "count", "itemCount", "collectionCount", "entryCount", "videoCount", "clipCount", "annotationCount", "packageCount", "sampledCount", "passedCount", "failedCount", "validCollectionCount", "invalidCollectionCount", "dataSource", "sourcePath", "bucket", "prefix", "endpoint", "pattern", "scanMode", "sourceFingerprint", "listedCount", "newCount", "rootPath", "segmentDurationSeconds", ): if key in payload: summary[key] = json_compatible(payload[key]) if "preview" in payload: summary["preview"] = json_compatible(payload.get("preview")) for key in dataset_partition_keys(payload): value = payload.get(key) if isinstance(value, list): summary[f"{key}Count"] = len(value) if "preview" not in summary: summary["preview"] = { "kind": "dataset", "format": str(payload.get("datasetType", "")).strip() or "json", "counts": {key: summary[key] for key in summary.keys() if key.endswith("Count") or key == "count"}, } return summary def chunk_list(values: list[Any], chunk_size: int = DATASET_PARTITION_SIZE) -> list[list[Any]]: if chunk_size <= 0: chunk_size = DATASET_PARTITION_SIZE return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] or [[]] def externalize_structured_payload(path: pathlib.Path, payload: dict[str, Any]) -> dict[str, Any]: normalized_payload = json_compatible(payload) dataset_dir = path.parent / "_datasets" / artifact_slug(path.stem or path.name) partitions_dir = dataset_dir / "partitions" partitions_dir.mkdir(parents=True, exist_ok=True) partition_files: dict[str, list[str]] = {} partition_count = 0 partition_keys = dataset_partition_keys(normalized_payload) for key in partition_keys: values = normalized_payload.get(key) if not isinstance(values, list): continue files: list[str] = [] for index, chunk in enumerate(chunk_list(values), start=1): partition_path = partitions_dir / f"{key}-p{index:06d}.json" write_json_file(partition_path, chunk) files.append(str(partition_path)) partition_files[key] = files partition_count += len(files) manifest = { "kind": "dataset", "datasetType": str(normalized_payload.get("datasetType", "")).strip() or "json", "payloadPath": str(path), "partitionDir": str(partitions_dir), "partitionKeys": partition_keys, "partitionFiles": partition_files, "summary": dataset_payload_summary(normalized_payload), "generatedAt": now_iso(), } manifest_path = write_json_file(dataset_dir / "manifest.json", manifest) data_ref = { "kind": "dataset", "datasetType": manifest["datasetType"], "manifestPath": str(manifest_path), "payloadPath": str(path), "partitionDir": str(partitions_dir), "partitionCount": partition_count, } item_count = 0 for key in partition_keys: values = normalized_payload.get(key) if isinstance(values, list): item_count += len(values) if item_count > 0: data_ref["itemCount"] = item_count summary = dataset_payload_summary(normalized_payload) summary["dataRef"] = data_ref return summary def write_json_artifact( run_id: str, node_id: str, output_dir: pathlib.Path, filename: str, payload: Any, artifact_type: str = "json", output: dict[str, Any] | None = None, ) -> dict[str, Any]: normalized_payload = json_compatible(payload) path = output_dir / filename path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: json.dump(normalized_payload, handle, ensure_ascii=False, indent=2) artifact_data: dict[str, Any] | None = normalize_json_payload(normalized_payload) if isinstance(normalized_payload, dict) and should_externalize_structured_payload(normalized_payload): artifact_data = externalize_structured_payload(path, normalized_payload) return build_artifact(run_id, node_id, path, artifact_type, data=artifact_data, output=output) def summarize_json_like(value: Any) -> dict[str, Any]: if isinstance(value, dict): return {"kind": "object", "keys": sorted(value.keys())} if isinstance(value, list): return {"kind": "list", "length": len(value)} return {"kind": type(value).__name__, "value": value} def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]: data = artifact.get("data") if isinstance(data, dict): data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {} payload_path = str(data_ref.get("payloadPath", "")).strip() if payload_path: payload = load_payload_from_file(pathlib.Path(payload_path)) if payload is not None: return normalize_json_payload(payload) if isinstance(data, dict): return data path_value = artifact.get("path") if isinstance(path_value, str) and path_value: payload = load_payload_from_file(pathlib.Path(path_value)) if payload is not None: return normalize_json_payload(payload) summary = { "name": artifact.get("name", ""), "type": artifact.get("type", ""), "path": artifact.get("path", ""), "size": artifact.get("size", ""), } return {key: value for key, value in summary.items() if value} def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]: merged: dict[str, Any] = {} for value in values: merged.update(value) return merged def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]: entries: list[dict[str, Any]] = [] if not base_path.exists(): return entries if base_path.is_file(): relative = base_path.name if pattern and not fnmatch.fnmatch(relative, pattern): return entries return [ { "path": relative, "sizeBytes": base_path.stat().st_size, "sha256": sha256_file(base_path), } ] for candidate in sorted(base_path.rglob("*")): if candidate.is_dir(): continue relative = candidate.relative_to(base_path).as_posix() if pattern and not fnmatch.fnmatch(relative, pattern): continue entries.append( { "path": relative, "sizeBytes": candidate.stat().st_size, "sha256": sha256_file(candidate), } ) return entries def list_bos_entries( bucket: str, prefix: str, pattern: str, endpoint: str, access_key_id: str, secret_access_key: str, ) -> list[dict[str, Any]]: client, scheme = build_bos_client(endpoint, access_key_id, secret_access_key) normalized_prefix = bos_prefix(prefix) prefix_with_slash = f"{normalized_prefix}/" if normalized_prefix else "" entries: list[dict[str, Any]] = [] for candidate in client.list_all_objects(bucket, prefix=normalized_prefix or None): key = text_value(getattr(candidate, "key", "")).strip().lstrip("/") if not key or key.endswith("/"): continue if normalized_prefix: if key == normalized_prefix: continue if not key.startswith(prefix_with_slash): continue relative_path = key[len(prefix_with_slash):] else: relative_path = key if not relative_path: continue if pattern and not fnmatch.fnmatch(relative_path, pattern): continue size_bytes = int(getattr(candidate, "size", 0) or 0) mime_type = mimetypes.guess_type(relative_path)[0] or "" storage_path = canonical_bos_path(bucket, key) entry: dict[str, Any] = { "path": relative_path, "storagePath": storage_path, "sizeBytes": size_bytes, "mimeType": mime_type, "etag": text_value(getattr(candidate, "e_tag", "") or getattr(candidate, "etag", "")), "versionId": text_value(getattr(candidate, "version_id", "") or getattr(candidate, "versionId", "")), "lastModified": text_value(getattr(candidate, "last_modified", "") or getattr(candidate, "lastModified", "")), "source": {"path": storage_path}, } file_name = pathlib.PurePosixPath(relative_path).name.lower() should_inline_json = relative_path.lower().endswith(".json") and ( file_name in XPLATFORM_INLINE_JSON_FILES or size_bytes <= PREVIEW_INLINE_BINARY_LIMIT ) if should_inline_json: raw = client.get_object_as_string(bucket, key) payload_text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw) try: entry["data"] = json.loads(payload_text) except Exception: entry["data"] = {"value": payload_text} elif pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"): entry["source"]["previewURL"] = client.generate_pre_signed_url( bucket, key, expiration_in_seconds=3600, protocol=scheme, ) entries.append(clean_metadata(entry) or {}) return entries def source_entry_fingerprint(data_source: str, bucket: str, entry: dict[str, Any]) -> str: normalized_source = (data_source or "BOS").strip().upper() or "BOS" relative_path = str(entry.get("path", "")).strip() version_id = str(entry.get("versionId", "")).strip() sha256 = str(entry.get("sha256", "")).strip() etag = str(entry.get("etag", "")).strip() size_bytes = str(entry.get("sizeBytes", "")).strip() last_modified = str(entry.get("lastModified", "")).strip() parts = [normalized_source, str(bucket or "").strip(), relative_path] if version_id: parts.append(f"version:{version_id}") elif sha256: parts.append(f"sha256:{sha256}") else: parts.extend([f"etag:{etag}", f"size:{size_bytes}", f"lastModified:{last_modified}"]) raw = "\n".join(parts).encode("utf-8") return hashlib.sha256(raw).hexdigest() def incremental_unit(params: dict[str, str]) -> str: value = (params.get("INCREMENTAL_UNIT") or "").strip().upper() if not value: return "OBJECT" if value in {"OBJECT", "DIRECTORY"}: return value raise RuntimeError("INCREMENTAL_UNIT must be OBJECT or DIRECTORY") def incremental_directory_depth(params: dict[str, str]) -> int: return parse_positive_int_param(params, "INCREMENTAL_DIRECTORY_DEPTH", 1) def directory_incremental_key(relative_path: str, depth: int) -> str: clean_relative = str(relative_path or "").strip().lstrip("/") if not clean_relative: return "__root__" parts = [segment for segment in pathlib.PurePosixPath(clean_relative).parts if segment not in ("", ".")] if len(parts) <= 1: return f"FILE::{clean_relative}" directory_parts = parts[:-1] take = min(max(depth, 1), len(directory_parts)) return "/".join(directory_parts[:take]) def incremental_source_state(params: dict[str, str]) -> dict[str, Any]: raw = params.get("SOURCE_STATE_JSON", "").strip() if not raw: return {"seenFingerprints": []} try: parsed = json.loads(raw) except Exception as exc: raise RuntimeError("SOURCE_STATE_JSON must be valid JSON") from exc if not isinstance(parsed, dict): raise RuntimeError("SOURCE_STATE_JSON must be a JSON object") seen = parsed.get("seenFingerprints") if seen is None: parsed["seenFingerprints"] = [] elif not isinstance(seen, list): raise RuntimeError("SOURCE_STATE_JSON.seenFingerprints must be an array") else: normalized_seen = [] for index, item in enumerate(seen): if not isinstance(item, str): raise RuntimeError(f"SOURCE_STATE_JSON.seenFingerprints[{index}] must be a string") fingerprint = item.strip() if fingerprint: normalized_seen.append(fingerprint) parsed["seenFingerprints"] = normalized_seen incremental_unit_value = str(parsed.get("incrementalUnit", "")).strip().upper() if incremental_unit_value and incremental_unit_value not in {"DIRECTORY", "OBJECT"}: raise RuntimeError("SOURCE_STATE_JSON.incrementalUnit must be OBJECT or DIRECTORY") parsed["incrementalUnit"] = incremental_unit_value or "OBJECT" directory_depth = parsed.get("directoryDepth", 1) if isinstance(directory_depth, bool) or not isinstance(directory_depth, int): raise RuntimeError("SOURCE_STATE_JSON.directoryDepth must be an integer") if directory_depth <= 0: raise RuntimeError("SOURCE_STATE_JSON.directoryDepth must be greater than 0") parsed["directoryDepth"] = directory_depth return parsed def apply_incremental_source_filter( entries: list[dict[str, Any]], data_source: str, bucket: str, params: dict[str, str], source_path: str, prefix: str, pattern: str, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: state = incremental_source_state(params) seen = {str(item).strip() for item in state.get("seenFingerprints", []) if str(item).strip()} scan_started_at = now_iso() mode = incremental_unit(params) depth = incremental_directory_depth(params) emitted: list[dict[str, Any]] = [] updated_seen = set(seen) if mode == "DIRECTORY": grouped_entries: dict[str, list[dict[str, Any]]] = {} for entry in entries: group_key = directory_incremental_key(str(entry.get("path", "")).strip(), depth) grouped_entries.setdefault(group_key, []).append(entry) for group_key in sorted(grouped_entries.keys()): group_entries = grouped_entries[group_key] if group_key in updated_seen: continue updated_seen.add(group_key) for entry in group_entries: enriched = dict(entry) enriched["fingerprint"] = source_entry_fingerprint(data_source, bucket, entry) enriched["incrementalKey"] = group_key emitted.append(enriched) else: for entry in entries: fingerprint = source_entry_fingerprint(data_source, bucket, entry) enriched = dict(entry) enriched["fingerprint"] = fingerprint enriched["incrementalKey"] = fingerprint if fingerprint in updated_seen: continue updated_seen.add(fingerprint) emitted.append(enriched) scan_completed_at = now_iso() source_fingerprint = str(params.get("SOURCE_FINGERPRINT", "")).strip() if not source_fingerprint: fingerprint_parts = [ "scanMode=INCREMENTAL", f"incrementalUnit={mode}", f"directoryDepth={depth}", f"dataSource={data_source.strip()}", f"sourcePath={source_path.strip()}", f"bucket={bucket.strip()}", f"prefix={prefix.strip()}", f"pattern={pattern.strip()}", f"endpoint={str(params.get('BOS_ENDPOINT', '')).strip()}", ] source_fingerprint = hashlib.sha256("\n".join(fingerprint_parts).encode("utf-8")).hexdigest() checkpoint = { "sourceFingerprint": source_fingerprint, "scanMode": "INCREMENTAL", "incrementalUnit": mode, "directoryDepth": depth, "dataSource": data_source, "sourcePath": source_path, "bucket": bucket, "prefix": prefix, "pattern": pattern, "seenFingerprints": sorted(updated_seen), "listedCount": len(entries), "newCount": len(emitted), "scanStartedAt": scan_started_at, "scanCompletedAt": scan_completed_at, } return emitted, checkpoint def preview_payload_for_remote_entry(relative_path: str, mime_type: str, preview_url: str) -> dict[str, Any] | None: normalized_preview_url = text_value(preview_url).strip() if not normalized_preview_url: return None suffix = pathlib.PurePosixPath(relative_path).suffix.lower() kind = "binary" if suffix in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"): kind = "video" elif suffix in IMAGE_FILE_SUFFIXES or mime_type.startswith("image/"): kind = "image" elif suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"): kind = "audio" return {"preview": {"kind": kind, "format": suffix.lstrip(".") or kind, "url": normalized_preview_url}} def source_entry_artifact( run_id: str, node_id: str, entry: dict[str, Any], source_path: str, output: dict[str, Any] | None = None, ) -> dict[str, Any]: relative_path = str(entry.get("path", "")).strip().lstrip("/") source = manifest_source_ref(entry, relative_path, source_path) storage_path = str(source.get("path", "")).strip() or source_path_join(source_path, relative_path) mime_type = str(entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or "" artifact_type = infer_port_type(pathlib.Path(relative_path)) payload: dict[str, Any] | None = None if "data" in entry: payload = normalize_json_payload(entry.get("data")) else: preview_payload = preview_payload_for_remote_entry(relative_path, mime_type, str(source.get("previewURL", "")).strip()) payload = preview_payload if preview_payload else None if payload is not None: payload["relativePath"] = relative_path if source: payload["source"] = source metadata = clean_metadata({"mimeType": mime_type, "sha256": entry.get("sha256", "")}) return build_virtual_artifact( run_id, node_id, pathlib.PurePosixPath(relative_path).name or "artifact", storage_path, artifact_type, int(entry.get("sizeBytes", 0) or 0), data=payload, metadata=metadata, output=output, ) def collect_output_artifacts( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any] | None = None, output: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: artifacts: list[dict[str, Any]] = [] for path in sorted(output_dir.rglob("*")): if path.is_dir(): continue artifact_type = infer_port_type(path) mime_type, _ = mimetypes.guess_type(path.name) if artifact_type == "json": payload = load_payload_from_file(path) else: payload = extract_preview_data(path, artifact_type, mime_type or "") binding = output if binding is None and node_data is not None: binding = output_binding(node_data) artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding)) return artifacts def hyperparams(node_data: dict[str, Any]) -> dict[str, str]: values: dict[str, str] = {} for item in node_data.get("hyperparams", []) or []: key = str(item.get("key", "")).strip() if not key: continue values[key] = str(item.get("value", "")) return values def parse_json_param(params: dict[str, str], key: str, default: Any, expected_type: type | tuple[type, ...] | None = None) -> Any: raw = params.get(key, "").strip() if not raw: return default try: parsed = json.loads(raw) except Exception as exc: raise RuntimeError(f"{key} must be valid JSON") from exc if expected_type is not None and not isinstance(parsed, expected_type): type_name = "array" if expected_type is list else "object" if expected_type is dict else "expected type" raise RuntimeError(f"{key} must be a JSON {type_name}") return parsed def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]: return [load_artifact_payload(item) for item in inputs] def collect_items(payloads: list[Any]) -> list[Any]: items: list[Any] = [] for payload in payloads: payload = normalize_json_payload(payload) if isinstance(payload.get("items"), list): items.extend(payload["items"]) elif isinstance(payload.get("entries"), list): items.extend(payload["entries"]) elif isinstance(payload.get("rows"), list): items.extend(payload["rows"]) else: items.append(payload) return items def normalize_mongo_document(item: Any) -> dict[str, Any]: if isinstance(item, dict): if set(item.keys()) == {"value"} and isinstance(item.get("value"), dict): return json_compatible(item["value"]) return json_compatible(item) return {"value": json_compatible(item)} def documents_from_inputs(payloads: list[dict[str, Any]], document_path: str) -> list[dict[str, Any]]: raw_items: list[Any] = [] path = str(document_path or "").strip() if path: for payload in payloads: value = get_field_value(payload, path) if isinstance(value, list): raw_items.extend(value) elif value is not None: raw_items.append(value) else: raw_items = collect_items(payloads) return [normalize_mongo_document(item) for item in raw_items] def build_mongo_client(uri: str): try: from pymongo import MongoClient except Exception as exc: raise RuntimeError("pymongo is required for MongoDB 写入算子") from exc return MongoClient(uri, serverSelectionTimeoutMS=5000) def get_field_value(value: Any, field_path: str) -> Any: current = value for token in [part for part in field_path.split(".") if part]: if isinstance(current, dict): current = current.get(token) continue if isinstance(current, list) and token.isdigit(): index = int(token) if 0 <= index < len(current): current = current[index] continue return None return current def normalize_comparison_operator(value: str, label: str = "OPERATOR", default: str = "eq") -> str: operator = str(value or "").strip().lower() or default if operator not in SUPPORTED_COMPARISON_OPERATORS: supported = ", ".join(sorted(SUPPORTED_COMPARISON_OPERATORS)) raise RuntimeError(f"{label} must be one of {supported}") return operator def normalize_sampling_rules(rules: list[Any], label: str = "RULES_JSON") -> list[dict[str, Any]]: normalized_rules: list[dict[str, Any]] = [] rule_ids: set[str] = set() for index, rule in enumerate(rules): if not isinstance(rule, dict): raise RuntimeError(f"{label}[{index}] must be an object") for key in ("id", "field", "operator", "samplePercent"): if key not in rule: raise RuntimeError(f"{label}[{index}].{key} is required") rule_id = str(rule.get("id", "")).strip() field_path = str(rule.get("field", "")).strip() if not rule_id: raise RuntimeError(f"{label}[{index}].id must be non-empty") if rule_id in rule_ids: raise RuntimeError(f"{label} contains duplicate id: {rule_id}") rule_ids.add(rule_id) if not field_path: raise RuntimeError(f"{label}[{index}].field must be non-empty") operator = normalize_comparison_operator(str(rule.get("operator", "")), f"{label}[{index}].operator") if operator not in {"exists", "missing"} and "value" not in rule: raise RuntimeError(f"{label}[{index}].value is required") sample_percent = parse_sample_percent(rule.get("samplePercent"), f"{label}[{index}].samplePercent") normalized_rules.append( { "id": rule_id, "field": field_path, "operator": operator, "value": rule.get("value"), "samplePercent": sample_percent, } ) return normalized_rules def parse_sample_percent(value: Any, label: str) -> float: if isinstance(value, bool): raise RuntimeError(f"{label} must be numeric") try: sample_percent = float(value) except Exception as exc: raise RuntimeError(f"{label} must be numeric") from exc if not math.isfinite(sample_percent): raise RuntimeError(f"{label} must be finite") if sample_percent <= 0 or sample_percent > 100: raise RuntimeError(f"{label} must be in the range (0, 100]") return sample_percent def normalize_field_map(field_map: dict[Any, Any], label: str = "FIELD_MAP_JSON") -> dict[str, str]: normalized: dict[str, str] = {} destinations: set[str] = set() for source, destination in field_map.items(): if not isinstance(source, str): raise RuntimeError(f"{label} source field names must be strings") source_name = source.strip() if not source_name: raise RuntimeError(f"{label} source field names must be non-empty") if source_name != source: raise RuntimeError(f"{label} source field {source!r} must not have leading or trailing whitespace") if not isinstance(destination, str): raise RuntimeError(f"{label}.{source_name} destination field must be a string") destination_name = destination.strip() if not destination_name: raise RuntimeError(f"{label}.{source_name} destination field must be non-empty") if destination_name != destination: raise RuntimeError(f"{label}.{source_name} destination field must not have leading or trailing whitespace") if destination_name in destinations: raise RuntimeError(f"{label} maps multiple source fields to destination field {destination_name!r}") destinations.add(destination_name) normalized[source_name] = destination_name return normalized def map_json_row(row: dict[str, Any], field_map: dict[str, str], label: str = "FIELD_MAP_JSON") -> dict[str, Any]: mapped: dict[str, Any] = {} mapped_sources: dict[str, str] = {} for key, value in row.items(): output_key = field_map.get(key, key) if output_key in mapped: previous_source = mapped_sources[output_key] raise RuntimeError(f"{label} maps source fields {previous_source!r} and {key!r} to the same output field {output_key!r}") mapped[output_key] = value mapped_sources[output_key] = key return mapped def compare_values(actual: Any, operator: str, expected: Any) -> bool: if operator == "exists": return actual is not None if operator == "missing": return actual is None if operator == "contains": if isinstance(actual, list): return expected in actual return str(expected) in str(actual) if operator == "in": if isinstance(expected, list): return actual in expected return str(actual) in str(expected) if operator == "regex": try: return re.search(str(expected), str(actual or "")) is not None except re.error as exc: raise RuntimeError(f"invalid regular expression: {exc}") from exc if operator in {"gt", "gte", "lt", "lte"}: try: actual_number = float(actual) expected_number = float(expected) except Exception: return False if operator == "gt": return actual_number > expected_number if operator == "gte": return actual_number >= expected_number if operator == "lt": return actual_number < expected_number return actual_number <= expected_number if operator == "ne": return actual != expected return actual == expected def parse_filter_dsl(value: str) -> list[tuple[str, str, Any]]: expression = str(value or "").strip() if not expression: return [] clauses: list[tuple[str, str, Any]] = [] operator_map = {"==": "eq", "!=": "ne", ">": "gt", ">=": "gte", "<": "lt", "<=": "lte", "~": "regex"} pattern = re.compile(r"^\s*([\w.\-\u4e00-\u9fff]+)\s*(<=|>=|==|!=|<|>|~)\s*(.+?)\s*$") for index, raw_clause in enumerate(expression.split("&")): match = pattern.match(raw_clause) if match is None: raise RuntimeError(f"DSL clause {index + 1} is invalid: {raw_clause.strip()}") field_path, operator_token, raw_expected = match.groups() expected_text = raw_expected.strip() if expected_text.endswith("s") and re.fullmatch(r"-?\d+(?:\.\d+)?s", expected_text): expected_text = expected_text[:-1] try: expected = json.loads(expected_text) except Exception: expected = expected_text.strip("\"'") clauses.append((field_path, operator_map[operator_token], expected)) return clauses def filter_rows(rows: list[dict[str, Any]], params: dict[str, str]) -> tuple[list[dict[str, Any]], dict[str, Any]]: dsl = params.get("DSL", "").strip() if dsl: clauses = parse_filter_dsl(dsl) filtered = [row for row in rows if all(compare_values(get_field_value(row, field), operator, expected) for field, operator, expected in clauses)] return filtered, {"dsl": dsl, "clauses": [{"field": field, "operator": operator, "expected": expected} for field, operator, expected in clauses]} field_path = params.get("FIELD_PATH", "").strip() operator = normalize_comparison_operator(params.get("OPERATOR", "eq")) expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", "")) return [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)], {"fieldPath": field_path, "operator": operator, "expected": expected} def flatten_context(value: Any, prefix: str = "") -> dict[str, str]: context: dict[str, str] = {} if isinstance(value, dict): for key, item in value.items(): next_prefix = f"{prefix}.{key}" if prefix else str(key) context.update(flatten_context(item, next_prefix)) if prefix: context[prefix] = json.dumps(value, ensure_ascii=False) return context if isinstance(value, list): for index, item in enumerate(value[:8]): next_prefix = f"{prefix}.{index}" if prefix else str(index) context.update(flatten_context(item, next_prefix)) if prefix: context[prefix] = json.dumps(value, ensure_ascii=False) return context if prefix: context[prefix] = "" if value is None else str(value) return context def render_template_text(template_text: str, context: dict[str, str]) -> str: rendered = template_text for key, value in context.items(): rendered = rendered.replace("{{" + key + "}}", value) safe_context = {key: value for key, value in context.items()} class SafeDict(dict): def __missing__(self, key: str) -> str: return "{" + key + "}" try: rendered = rendered.format_map(SafeDict(safe_context)) except Exception: pass return rendered def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for item in collect_items(payloads): if isinstance(item, dict): rows.append(item) else: rows.append({"value": item}) return rows def write_text_artifact( run_id: str, node_id: str, output_dir: pathlib.Path, filename: str, content: str, output: dict[str, Any] | None = None, ) -> dict[str, Any]: path = output_dir / filename path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") mime_type, _ = mimetypes.guess_type(path.name) preview = extract_preview_data(path, infer_port_type(path), mime_type or "") return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output) def write_csv_artifact( run_id: str, node_id: str, output_dir: pathlib.Path, filename: str, rows: list[dict[str, Any]], output: dict[str, Any] | None = None, ) -> dict[str, Any]: path = output_dir / filename path.parent.mkdir(parents=True, exist_ok=True) columns: list[str] = [] for row in rows: for key in row.keys(): if key not in columns: columns.append(key) with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=columns or ["value"]) writer.writeheader() if rows: writer.writerows(rows) preview = preview_tabular_payload(path, ",", "csv") return build_artifact(run_id, node_id, path, "file", data=preview, output=output) def write_video_segment(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]: cv2, _ = require_cv2() capture = cv2.VideoCapture(str(source_path)) if not capture.isOpened(): raise RuntimeError(f"无法打开视频文件: {source_path}") try: fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0 width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64 height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64 frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) start_frame = max(0, int(start_second * fps)) end_frame = max(start_frame + 1, int(math.ceil(end_second * fps))) if frame_count > 0: end_frame = min(end_frame, frame_count) capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame) destination.parent.mkdir(parents=True, exist_ok=True) writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) if not writer.isOpened(): raise RuntimeError(f"无法创建视频切片: {destination}") written = 0 try: for _ in range(start_frame, end_frame): ok, frame = capture.read() if not ok: break writer.write(frame) written += 1 finally: writer.release() duration = written / fps if fps > 0 else 0.0 return { "frameRate": round(fps, 3), "frameCount": written, "resolution": f"{width}x{height}", "duration": round(duration, 3), } finally: capture.release() def ffmpeg_available() -> bool: return shutil.which("ffmpeg") is not None def run_ffmpeg(args: list[str]) -> None: completed = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if completed.returncode == 0: return stderr = (completed.stderr or "").strip() stdout = (completed.stdout or "").strip() message = stderr or stdout or "unknown ffmpeg error" raise RuntimeError(f"ffmpeg execution failed: {message}") def write_video_segment_with_ffmpeg(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]: if not ffmpeg_available(): raise RuntimeError("ffmpeg is not available in operator-runner image") duration = max(float(end_second) - float(start_second), 0.001) destination.parent.mkdir(parents=True, exist_ok=True) copy_args = [ "ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-ss", f"{max(float(start_second), 0.0):.3f}", "-i", str(source_path), "-t", f"{duration:.3f}", "-map", "0:v:0", "-map", "0:a?", "-c", "copy", str(destination), ] reencode_args = [ "ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-ss", f"{max(float(start_second), 0.0):.3f}", "-i", str(source_path), "-t", f"{duration:.3f}", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p", "-c:a", "aac", str(destination), ] try: run_ffmpeg(copy_args) except RuntimeError: run_ffmpeg(reencode_args) metrics = inspect_video_file(destination) return { "frameRate": metrics.get("frameRate"), "frameCount": metrics.get("frameCount"), "resolution": metrics.get("resolution"), "duration": metrics.get("duration", round(duration, 3)), } def write_video_segments( source_path: pathlib.Path | str, segments: list[dict[str, Any]], ) -> list[dict[str, Any]]: cv2, _ = require_cv2() capture = cv2.VideoCapture(str(source_path)) if not capture.isOpened(): raise RuntimeError(f"无法打开视频文件: {source_path}") try: fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0 width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64 height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64 frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) normalized_segments: list[dict[str, Any]] = [] for raw in segments: destination = pathlib.Path(raw.get("destination", "")) start_second = float(raw.get("startSecond") or 0.0) end_second = float(raw.get("endSecond") or 0.0) start_frame = max(0, int(start_second * fps)) end_frame = max(start_frame + 1, int(math.ceil(end_second * fps))) if frame_count > 0: end_frame = min(end_frame, frame_count) normalized_segments.append( { **raw, "destination": destination, "startFrame": start_frame, "endFrame": end_frame, } ) normalized_segments.sort(key=lambda item: (int(item["startFrame"]), str(item["destination"]))) current_frame = 0 if normalized_segments: initial_frame = int(normalized_segments[0]["startFrame"]) if initial_frame > 0: capture.set(cv2.CAP_PROP_POS_FRAMES, initial_frame) current_frame = initial_frame results: list[dict[str, Any]] = [] for segment in normalized_segments: start_frame = int(segment["startFrame"]) end_frame = int(segment["endFrame"]) destination = pathlib.Path(segment["destination"]) if current_frame < start_frame: while current_frame < start_frame: ok, _ = capture.read() if not ok: current_frame = end_frame break current_frame += 1 destination.parent.mkdir(parents=True, exist_ok=True) writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) if not writer.isOpened(): raise RuntimeError(f"无法创建视频切片: {destination}") written = 0 try: while current_frame < end_frame: ok, frame = capture.read() if not ok: break writer.write(frame) written += 1 current_frame += 1 finally: writer.release() results.append( { **segment, "frameRate": round(fps, 3), "frameCount": written, "resolution": f"{width}x{height}", "duration": round(written / fps, 3) if fps > 0 else 0.0, } ) return results finally: capture.release() def write_video_segment_from_remote_url(source_url: str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]: return write_video_segment(source_url, destination, start_second, end_second) def annotate_video_clip(path: pathlib.Path, frame_interval_seconds: float) -> dict[str, Any]: cv2, np = require_cv2() capture = cv2.VideoCapture(str(path)) if not capture.isOpened(): raise RuntimeError(f"无法打开视频切片: {path}") try: fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0 frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0) duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0 step = max(1, int(round(max(frame_interval_seconds, 0.1) * fps))) previous_gray = None samples: list[tuple[float, float]] = [] objects: set[str] = set() frame_index = 0 while True: ok, frame = capture.read() if not ok: break if frame_index % step != 0: frame_index += 1 continue timestamp = frame_index / fps if fps > 0 else 0.0 gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) motion = 0.0 if previous_gray is not None: diff = cv2.absdiff(previous_gray, gray) motion = float(diff.mean() / 255.0) previous_gray = gray ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb) mask = cv2.inRange(ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8)) if float(mask.mean() / 255.0) > 0.1: objects.add("手部") _, threshold = cv2.threshold(gray, 24, 255, cv2.THRESH_BINARY) contours, _ = cv2.findContours(threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if any(cv2.contourArea(contour) >= 64 for contour in contours): objects.add("前景目标") samples.append((timestamp, motion)) frame_index += 1 if not samples: samples.append((0.0, 0.0)) def label_for_motion(motion: float) -> tuple[str, float]: if motion < 0.015: return "静止观察", 0.88 if motion < 0.045: return "平稳活动", 0.79 return "快速移动", 0.71 actions: list[dict[str, Any]] = [] last_timestamp = duration for timestamp, motion in samples: label, confidence = label_for_motion(motion) last_timestamp = timestamp if not actions: actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)}) continue current = actions[-1] if current["label"] != label: current["end"] = round(timestamp, 3) actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)}) else: current["end"] = round(timestamp, 3) current["confidence"] = round(min(float(current["confidence"]), confidence), 3) if actions: actions[-1]["end"] = round(max(duration, last_timestamp), 3) min_confidence = min(float(item["confidence"]) for item in actions) if actions else 0.0 return { "duration": round(duration, 3), "actions": actions, "objects": sorted(objects), "minConfidence": round(min_confidence, 3), "frameIntervalSeconds": frame_interval_seconds, } finally: capture.release() def builtin_source( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT")) data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper() if data_source and data_source not in VALID_SOURCE_DATA_SOURCES: raise RuntimeError("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM") source_path = (params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "")).strip() prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip() bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip() pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip() bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip() bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip() bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip() scan_mode = (params.get("SCAN_MODE") or "").strip().upper() if scan_mode and scan_mode not in {"FULL", "INCREMENTAL"}: raise RuntimeError("SCAN_MODE must be FULL or INCREMENTAL") unit = incremental_unit(params) depth = incremental_directory_depth(params) source_bucket, source_prefix = parse_bos_path(source_path) if not bucket and source_bucket: bucket = source_bucket if not prefix and source_prefix: prefix = source_prefix local_source_path = pathlib.Path(source_path) if source_path and not source_bucket else None local_prefix_path = pathlib.Path(prefix) if prefix and "://" not in prefix else None has_bos_connection = bool(bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key) if inline_source: data_source = "INLINE" elif not data_source: if source_bucket or has_bos_connection: data_source = "BOS" elif local_source_path is not None: data_source = "LOCAL" elif local_prefix_path is not None and local_prefix_path.exists(): data_source = "LOCAL" elif inputs: data_source = "UPSTREAM" if data_source == "INLINE" and not inline_source: raise RuntimeError("真实数据源 DATA_SOURCE=INLINE requires SOURCE_TEXT or SOURCE_JSON") if data_source == "BOS": missing = [] if not bucket: missing.append("bucket or bos:// SOURCE_PATH") if not bos_endpoint: missing.append("BOS_ENDPOINT") if not bos_access_key_id: missing.append("BOS_ACCESS_KEY_ID") if not bos_secret_access_key: missing.append("BOS_SECRET_ACCESS_KEY") if missing: raise RuntimeError("真实数据源 DATA_SOURCE=BOS requires " + ", ".join(missing)) if data_source == "LOCAL": if source_bucket: raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL cannot read a bos:// SOURCE_PATH") if local_source_path is not None and not local_source_path.exists(): raise RuntimeError(f"真实数据源 SOURCE_PATH does not exist inside operator-runner: {source_path}") if local_source_path is None and (local_prefix_path is None or not local_prefix_path.exists()): raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL requires SOURCE_PATH or an existing STORAGE_PREFIX path") if not data_source: raise RuntimeError( "真实数据源 requires SOURCE_TEXT, SOURCE_JSON, SOURCE_PATH, a configured BOS source, " "an existing local STORAGE_PREFIX path, or upstream input artifacts" ) entries: list[dict[str, Any]] = [] resolved_source_path = source_path if params.get("SOURCE_JSON"): parsed = json.loads(params["SOURCE_JSON"]) if isinstance(parsed, list): entries = [{"value": item} for item in parsed] else: entries = [{"value": parsed}] elif params.get("SOURCE_TEXT"): entries = [{"text": params["SOURCE_TEXT"]}] elif data_source == "BOS": cleaned_prefix = bos_prefix(prefix) entries = list_bos_entries( bucket, cleaned_prefix, pattern, bos_endpoint, bos_access_key_id, bos_secret_access_key, ) resolved_source_path = canonical_bos_path(bucket, cleaned_prefix) elif local_source_path is not None: entries = list_source_entries(local_source_path, pattern) elif local_prefix_path is not None and local_prefix_path.exists(): entries = list_source_entries(local_prefix_path, pattern) elif inputs: entries = [load_artifact_payload(item) for item in inputs] listed_entries = list(entries) checkpoint_payload = None if scan_mode == "INCREMENTAL": entries, checkpoint_payload = apply_incremental_source_filter( listed_entries, data_source, bucket, params, resolved_source_path, prefix, pattern, ) payload = { "dataSource": data_source, "bucket": bucket, "prefix": prefix, "endpoint": bos_endpoint, "pattern": pattern, "scanMode": scan_mode or "FULL", "incrementalUnit": unit, "directoryDepth": depth, "sourcePath": resolved_source_path, "sourceFingerprint": str((checkpoint_payload or {}).get("sourceFingerprint") or params.get("SOURCE_FINGERPRINT", "")).strip(), "listedCount": len(listed_entries), "newCount": len(entries) if checkpoint_payload is not None else len(entries), "entries": entries, "catalogEntries": listed_entries, } artifacts = [ write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data)) ] if checkpoint_payload is not None: artifacts.append(write_json_artifact(run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload)) if data_source == "BOS": for entry in entries: artifacts.append(source_entry_artifact(run_id, node_id, entry, resolved_source_path, output=output_binding(node_data))) elif source_path and pathlib.Path(source_path).exists(): source_root = pathlib.Path(source_path) staged_root = output_dir / "_source_tree" for entry in entries: relative_path = str(entry.get("path", "")).strip() if not relative_path: continue candidate = source_root / relative_path if source_root.is_dir() else source_root if not candidate.exists() or not candidate.is_file(): continue destination = staged_root / relative_path destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(candidate, destination) artifacts.extend(collect_output_artifacts(run_id, node_id, staged_root, node_data)) return artifacts def builtin_merge( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: payloads = payloads_from_inputs(inputs) merged = merge_dicts(payloads) merged["inputCount"] = len(payloads) return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))] def builtin_intersection( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: payloads = payloads_from_inputs(inputs) if not payloads: return [ write_json_artifact( run_id, node_id, output_dir, "intersection.json", {"keys": [], "inputCount": 0}, output=output_binding(node_data), ) ] key_sets = [set(payload.keys()) for payload in payloads] common_keys = sorted(set.intersection(*key_sets)) if key_sets else [] payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}} return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))] def builtin_distribution( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: payloads = payloads_from_inputs(inputs) items = collect_items(payloads) outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}] partitions: list[list[Any]] = [[] for _ in outputs] for index, item in enumerate(items): partitions[index % len(outputs)].append(item) artifacts: list[dict[str, Any]] = [] for index, output in enumerate(outputs): filename = f"{output.get('name', f'queue-{index}')}.json" payload = {"items": partitions[index], "count": len(partitions[index])} artifacts.append( write_json_artifact( run_id, node_id, output_dir, filename, payload, str(output.get("type", "json")), output=output, ) ) return artifacts def builtin_upload( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip() bucket = str(node_data.get("bucket", "")).strip() prefix = str(node_data.get("prefix", "")).strip() parsed_bucket, parsed_prefix = parse_bos_path(upload_path) if parsed_bucket: bucket = parsed_bucket if parsed_prefix or upload_path.endswith("/"): prefix = parsed_prefix data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper() bos_endpoint = params.get("BOS_ENDPOINT", "").strip() bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip() bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip() copied_paths: list[str] = [] bundle_inputs: list[dict[str, Any]] = [] staging_dir = output_dir / "uploaded" staging_dir.mkdir(parents=True, exist_ok=True) package_manifest_payload = first_payload_with_keys(inputs, "packageCount", "items") upload_inputs = list(inputs) if package_manifest_payload is not None: package_names = package_related_artifact_names(package_manifest_payload) upload_inputs = [ artifact for artifact in inputs if str(artifact.get("name", "")).strip() not in package_names ] upload_inputs.extend( materialize_package_artifacts_from_manifest( run_id, node_id, staging_dir, package_manifest_payload, node_data, ) ) if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key: client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key) temp_paths = [ str(item.get("path", "")).strip() for item in upload_inputs if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip() ] input_root = common_input_root(temp_paths) uploaded_objects: list[dict[str, Any]] = [] remote_artifacts: list[dict[str, Any]] = [] for index, artifact in enumerate(upload_inputs): payload = load_artifact_payload(artifact) bundle_inputs.append(payload) fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}" relative_path = upload_relative_path(artifact, input_root, fallback_name) temp_source = materialized_upload_source(artifact, staging_dir, relative_path) if temp_source is None or not temp_source.exists(): continue mime_type = ( str((artifact.get("metadata") or {}).get("mimeType", "")).strip() if isinstance(artifact.get("metadata"), dict) else "" ) or mimetypes.guess_type(temp_source.name)[0] or "" object_key = bos_key_join(prefix, relative_path) put_kwargs = {"content_type": mime_type} if mime_type else {} try: size_bytes = temp_source.stat().st_size except OSError: size_bytes = 0 log( f"[operator-runner][info] upload starts: {temp_source} -> {canonical_bos_path(bucket, object_key)} " f"({size_bytes} bytes)" ) client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs) remote_path = canonical_bos_path(bucket, object_key) log( f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} " f"({size_bytes} bytes)" ) preview_url = "" if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"): preview_url = text_value( client.generate_pre_signed_url( bucket, object_key, expiration_in_seconds=3600, protocol=scheme, ) ).strip() uploaded_objects.append( clean_metadata( { "name": pathlib.PurePosixPath(relative_path).name, "relativePath": relative_path, "path": remote_path, "previewURL": preview_url, "sizeBytes": temp_source.stat().st_size, "mimeType": mime_type, } ) or {} ) artifact_type = str(artifact.get("type", "")).strip() or infer_port_type(temp_source) artifact_payload: dict[str, Any] | None = None if artifact_type == "json": loaded = load_payload_from_file(temp_source) if loaded is not None: artifact_payload = normalize_json_payload(loaded) else: artifact_payload = preview_payload_for_remote_entry(relative_path, mime_type, preview_url) if artifact_payload is None: artifact_payload = extract_preview_data(temp_source, artifact_type, mime_type) if artifact_payload is not None: artifact_payload["relativePath"] = relative_path artifact_payload["source"] = clean_metadata( { "path": remote_path, "previewURL": preview_url, "relativePath": relative_path, "mimeType": mime_type, } ) or {} source_size_bytes = temp_source.stat().st_size source_sha256, source_sha256_metadata = artifact_sha256(temp_source, source_size_bytes) remote_artifacts.append( build_virtual_artifact( run_id, node_id, pathlib.PurePosixPath(relative_path).name, remote_path, artifact_type, source_size_bytes, data=artifact_payload, metadata={ **source_sha256_metadata, "sha256": source_sha256, "mimeType": mime_type, }, output=output_binding(node_data), ) ) payload = { "copiedPaths": copied_paths, "uploadedCount": len(uploaded_objects), "uploadedObjects": uploaded_objects, "items": bundle_inputs, "inputCount": len(inputs), "dataSource": "BOS", "bucket": bucket, "prefix": prefix, "destinationPath": canonical_bos_path(bucket, prefix), } artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))] artifacts.extend(remote_artifacts) return artifacts for artifact in upload_inputs: payload = load_artifact_payload(artifact) bundle_inputs.append(payload) source_path = artifact.get("path") if isinstance(source_path, str) and source_path and os.path.exists(source_path): source = pathlib.Path(source_path) destination = staging_dir / source.name if source.resolve() != destination.resolve(): shutil.copy2(source, destination) copied_paths.append(str(destination)) payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)} artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))] artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir)) return artifacts def builtin_json_extract( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) field_path = params.get("FIELD_PATH", "").strip() payloads = payloads_from_inputs(inputs) payload = payloads[0] if payloads else {} extracted = get_field_value(payload, field_path) if field_path else payload result = {"fieldPath": field_path, "value": extracted} return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))] def builtin_json_filter( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) rows = collect_items(payloads_from_inputs(inputs)) filtered, filter_metadata = filter_rows(rows, params) payload = { **filter_metadata, "items": filtered, "count": len(filtered), } return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))] def builtin_json_map( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) field_map = normalize_field_map(parse_json_param(params, "FIELD_MAP_JSON", {}, dict)) rows = rows_from_payloads(payloads_from_inputs(inputs)) mapped = [map_json_row(row, field_map) for row in rows] payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)} return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))] def builtin_mongodb_write( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) mongo_target = params.get("MONGO_TARGET", "").strip().lower() or "business" explicit_mongo_uri = params.get("MONGO_URI", "").strip() or os.getenv("MONGO_URI", "").strip() explicit_mongo_db = params.get("MONGO_DB", "").strip() or os.getenv("MONGO_DB", "").strip() collection_name = params.get("MONGO_COLLECTION", "").strip() mode = params.get("MONGO_MODE", "insert_many").strip().lower() or "insert_many" upsert_key = params.get("UPSERT_KEY", "_id").strip() or "_id" document_path = params.get("DOCUMENT_PATH", "").strip() mongo_uri = explicit_mongo_uri mongo_db = explicit_mongo_db if not mongo_uri: if mongo_target == "platform": mongo_uri = os.getenv("PLATFORM_MONGO_URI", "").strip() mongo_db = mongo_db or os.getenv("PLATFORM_MONGO_DB", "").strip() else: mongo_uri = os.getenv("BUSINESS_MONGO_URI", "").strip() mongo_db = mongo_db or os.getenv("BUSINESS_MONGO_DB", "").strip() if not mongo_uri: raise RuntimeError(f"MongoDB 写入缺少可用连接:target={mongo_target}") if not mongo_db: raise RuntimeError(f"MongoDB 写入缺少数据库名:target={mongo_target}") if not collection_name: raise RuntimeError("MONGO_COLLECTION is required for MongoDB 写入") if mongo_target == "platform" and collection_name.lower() in {item.lower() for item in RESERVED_MONGO_COLLECTIONS}: raise RuntimeError(f"collection {collection_name} is reserved and cannot be written by workflow operators") if mode not in {"insert_many", "upsert"}: raise RuntimeError("MONGO_MODE must be insert_many or upsert") documents = documents_from_inputs(payloads_from_inputs(inputs), document_path) if not documents: raise RuntimeError("MongoDB 写入未收到可写入的文档") client = build_mongo_client(mongo_uri) try: collection = client[mongo_db][collection_name] summary: dict[str, Any] = { "target": mongo_target, "database": mongo_db, "collection": collection_name, "mode": mode, "documentCount": len(documents), } if document_path: summary["documentPath"] = document_path if mode == "insert_many": result = collection.insert_many(documents, ordered=False) summary["insertedCount"] = len(getattr(result, "inserted_ids", []) or []) else: processed = 0 matched_count = 0 modified_count = 0 upserted_ids: list[Any] = [] for document in documents: key_value = get_field_value(document, upsert_key) if key_value is None: raise RuntimeError(f"document is missing UPSERT_KEY={upsert_key}") result = collection.replace_one({upsert_key: key_value}, document, upsert=True) processed += 1 matched_count += int(getattr(result, "matched_count", 0) or 0) modified_count += int(getattr(result, "modified_count", 0) or 0) upserted_id = getattr(result, "upserted_id", None) if upserted_id is not None: upserted_ids.append(json_compatible(upserted_id)) summary["upsertKey"] = upsert_key summary["upsertedCount"] = processed summary["matchedCount"] = matched_count summary["modifiedCount"] = modified_count if upserted_ids: summary["upsertedIds"] = upserted_ids return [ write_json_artifact( run_id, node_id, output_dir, "mongodb-write-result.json", summary, output=output_binding(node_data), ) ] finally: close = getattr(client, "close", None) if callable(close): close() def builtin_text_template( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) template_text = params.get("TEMPLATE_TEXT", "").strip() output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt" payloads = payloads_from_inputs(inputs) context = flatten_context(payloads[0] if payloads else {}) rendered = render_template_text(template_text, context) binding = output_binding(node_data) manifest = write_json_artifact( run_id, node_id, output_dir, "rendered-template.json", {"template": template_text, "context": context, "outputName": output_name}, output=binding, ) return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)] def builtin_csv_to_json( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) source_path = resolve_source_path(params, inputs) payload: dict[str, Any] if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES: delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()] columns, rows = load_tabular_rows(source_path, delimiter) payload = { "sourcePath": str(source_path), "columns": columns, "items": rows, "rowCount": len(rows), } else: for artifact in inputs: preview = artifact_preview_payload(artifact) or {} if preview.get("kind") == "table": payload = { "sourcePath": str(artifact.get("path", "")), "columns": preview.get("columns", []), "items": preview.get("rows", []), "rowCount": preview.get("rowCount", len(preview.get("rows", []))), "previewTruncated": preview.get("truncated", False), } return [ write_json_artifact( run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data), ) ] raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件") return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))] def builtin_json_to_csv( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: rows = rows_from_payloads(payloads_from_inputs(inputs)) binding = output_binding(node_data) summary = write_json_artifact( run_id, node_id, output_dir, "csv-export-summary.json", {"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})}, output=binding, ) return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)] def builtin_directory_scan( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) source_path = resolve_source_path(params, inputs) if source_path is not None: root = source_path if source_path.is_dir() else source_path.parent payload = { "sourcePath": str(source_path), "scanRoot": str(root), "summary": summarize_directory_tree(root), } elif inputs: payload = { "sourcePath": "", "scanRoot": "", "summary": summarize_input_artifacts(inputs), } else: raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录") return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))] def builtin_xplatform_index( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: payload = load_xplatform_index_payload(hyperparams(node_data), inputs, output_dir) return [write_json_artifact(run_id, node_id, output_dir, "dataset-index.json", payload, output=output_binding(node_data))] def builtin_collection_validate( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) require_intrinsics = parse_bool_param(params, "REQUIRE_INTRINSICS", False) payload = first_payload_with_keys(inputs, "collections", "items") if payload is None: payload = load_xplatform_index_payload(params, inputs, output_dir) collections = [] valid_collection_count = 0 for collection in payload.get("collections", []): if not isinstance(collection, dict): continue missing_required = list(collection.get("missingRequiredFiles", [])) if require_intrinsics and not collection.get("hasIntrinsics", False): missing_required.append("intrinsics.json") valid = len(missing_required) == 0 and int(collection.get("videoCount", 0) or 0) > 0 if valid: valid_collection_count += 1 collections.append({**collection, "missingRequiredFiles": missing_required, "valid": valid}) items = [] for item in payload.get("items", []): if not isinstance(item, dict): continue collection_id = str(item.get("collectionId", "")).strip() items.append({**item, "collectionValid": any(c.get("id") == collection_id and c.get("valid", False) for c in collections)}) result = { **payload, "datasetType": "x-platform-validation", "collections": collections, "items": items, "validCollectionCount": valid_collection_count, "invalidCollectionCount": max(len(collections) - valid_collection_count, 0), } return [write_json_artifact(run_id, node_id, output_dir, "dataset-validation.json", result, output=output_binding(node_data))] def builtin_video_quality( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) min_clarity = parse_unit_interval_param(params, "MIN_CLARITY", DEFAULT_MIN_CLARITY) max_shake = parse_unit_interval_param(params, "MAX_SHAKE", DEFAULT_MAX_SHAKE) max_hand_occlusion = parse_unit_interval_param(params, "MAX_HAND_OCCLUSION", DEFAULT_MAX_HAND_OCCLUSION) clarity_scale = parse_positive_float_param(params, "CLARITY_SCALE", 35.0) payload = first_payload_with_keys(inputs, "items") if payload is None: payload = load_xplatform_index_payload(params, inputs, output_dir) archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} items = [] passed_count = 0 for item in payload.get("items", []): if not isinstance(item, dict): continue source_path = source_path_from_item(item, str(item.get("filename", "video.mp4")), output_dir, archive_sources) metrics = analyze_video_quality_metrics(source_path, clarity_scale) passed = ( float_value(metrics.get("clarity"), 0.0) >= min_clarity and float_value(metrics.get("shake"), 1.0) <= max_shake and float_value(metrics.get("handOcclusion"), 1.0) <= max_hand_occlusion ) if passed: passed_count += 1 items.append( { **item, "quality": { **metrics, "passed": passed, "thresholds": { "minClarity": min_clarity, "maxShake": max_shake, "maxHandOcclusion": max_hand_occlusion, }, }, } ) result = { **payload, "datasetType": "x-platform-quality", "items": items, "videoCount": len(items), "passedCount": passed_count, "failedCount": max(len(items) - passed_count, 0), } return [write_json_artifact(run_id, node_id, output_dir, "quality-report.json", result, output=output_binding(node_data))] def builtin_dataset_filter( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) payload = first_payload_with_keys(inputs, "items") if payload is None: raise RuntimeError("数据过滤需要上游数据清单输入") date_start = normalize_date_value(params.get("DATE_START", "")) date_end = normalize_date_value(params.get("DATE_END", "")) place = params.get("PLACE", "").strip() collection_type = params.get("TYPE", "").strip() min_clarity = parse_unit_interval_param(params, "MIN_CLARITY", DEFAULT_MIN_CLARITY) max_shake = parse_unit_interval_param(params, "MAX_SHAKE", DEFAULT_MAX_SHAKE) max_hand_occlusion = parse_unit_interval_param(params, "MAX_HAND_OCCLUSION", DEFAULT_MAX_HAND_OCCLUSION) filtered = [] for item in payload.get("items", []): if not isinstance(item, dict): continue meta = item.get("meta") if isinstance(item.get("meta"), dict) else {} quality = item.get("quality") if isinstance(item.get("quality"), dict) else {} if place and str(meta.get("place", "")).strip() != place: continue if collection_type and str(meta.get("type", "")).strip() != collection_type: continue if date_start or date_end: if not date_in_range(meta.get("date", ""), date_start, date_end): continue if float_value(quality.get("clarity"), 0.0) < min_clarity: continue if float_value(quality.get("shake"), 1.0) > max_shake: continue if float_value(quality.get("handOcclusion"), 1.0) > max_hand_occlusion: continue if not bool(quality.get("passed", False)): continue filtered.append(item) result = { "datasetType": "x-platform-filtered", "criteria": { "dateStart": date_start, "dateEnd": date_end, "place": place, "type": collection_type, "minClarity": min_clarity, "maxShake": max_shake, "maxHandOcclusion": max_hand_occlusion, }, "count": len(filtered), "items": filtered, } if isinstance(payload.get("archiveSources"), dict): result["archiveSources"] = payload["archiveSources"] return [write_json_artifact(run_id, node_id, output_dir, "filtered-videos.json", result, output=output_binding(node_data))] def builtin_video_slicing( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) payload = first_payload_with_keys(inputs, "items") if payload is None: raise RuntimeError("视频切片需要上游过滤结果输入") segment_duration = parse_positive_int_param(params, "SEGMENT_DURATION_SECONDS", 300) archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} clips: list[dict[str, Any]] = [] for item in payload.get("items", []): if not isinstance(item, dict): continue source_filename = str(item.get("filename", "")).strip() or "video.mp4" source_path = source_path_from_item(item, source_filename, output_dir, archive_sources) metrics = inspect_video_file(source_path) duration = float(item.get("duration") or metrics.get("duration") or 0.0) if duration <= 0: duration = float(metrics.get("duration") or 0.0) if duration <= 0: duration = float(segment_duration) clip_count = max(1, int(math.ceil(duration / segment_duration))) stem = pathlib.Path(source_filename).stem clip_specs: list[dict[str, Any]] = [] for clip_index in range(clip_count): clip_start = clip_index * segment_duration clip_end = min(duration, clip_start + segment_duration) clip_name = f"{stem}_clip{clip_index + 1:03d}.mp4" clip_path = output_dir / "clips" / clip_name clip_specs.append( { "clipIndex": clip_index, "clipStart": clip_start, "clipEnd": clip_end, "clipName": clip_name, "clipPath": clip_path, "destination": clip_path, "startSecond": clip_start, "endSecond": clip_end, } ) clip_metrics_list: list[dict[str, Any]] | None = None if ffmpeg_available(): try: log(f"[operator-runner][info] video_slicing uses ffmpeg for {source_filename} ({clip_count} clip(s))") clip_metrics_list = [] for clip_spec in clip_specs: clip_metrics_list.append( { **clip_spec, **write_video_segment_with_ffmpeg( source_path, pathlib.Path(clip_spec["clipPath"]), float(clip_spec["clipStart"]), float(clip_spec["clipEnd"]), ), } ) except RuntimeError as exc: log(f"[operator-runner][warn] ffmpeg video slicing failed for {source_filename}, fallback to opencv: {exc}") clip_metrics_list = None if clip_metrics_list is None: if len(clip_specs) > 1: clip_metrics_list = write_video_segments(source_path, clip_specs) else: only = clip_specs[0] clip_metrics_list = [ { **only, **write_video_segment(source_path, only["clipPath"], float(only["clipStart"]), float(only["clipEnd"])), } ] for clip_spec in clip_metrics_list: clip_start = float(clip_spec["clipStart"]) clip_end = float(clip_spec["clipEnd"]) clip_name = str(clip_spec["clipName"]) clip_path = pathlib.Path(clip_spec["clipPath"]) clips.append( { "clipId": pathlib.Path(clip_name).stem, "clipName": clip_name, "clipPath": str(clip_path), "sourceVideo": source_filename, "sourcePath": str(source_path), "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, "collectionId": item.get("collectionId", ""), "meta": item.get("meta", {}), "quality": item.get("quality", {}), "startSecond": round(clip_start, 3), "endSecond": round(clip_end, 3), "duration": clip_spec.get("duration", round(max(clip_end - clip_start, 0.0), 3)), "resolution": clip_spec.get("resolution"), "frameRate": clip_spec.get("frameRate"), } ) manifest = { "datasetType": "x-platform-clips", "segmentDurationSeconds": segment_duration, "clipCount": len(clips), "items": clips, } artifacts = [write_json_artifact(run_id, node_id, output_dir, "clips-manifest.json", manifest, output=output_binding(node_data))] artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "clips", node_data=node_data, output=output_binding(node_data))) return artifacts def builtin_video_annotation( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) payload = first_payload_with_keys(inputs, "clipCount", "items") if payload is None: raise RuntimeError("视频标注需要上游切片结果输入") frame_interval = parse_positive_float_param(params, "FRAME_INTERVAL_SECONDS", 1.0) clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")} archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} items = [] for clip in payload.get("items", []): if not isinstance(clip, dict): continue clip_name = str(clip.get("clipName", "")).strip() or pathlib.Path(str(clip.get("clipPath", ""))).name artifact = clip_artifacts.get(clip_name) clip_path: pathlib.Path | None = None if artifact is not None: try: clip_path = source_path_from_item( {"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")}, clip_name, output_dir, archive_sources, ) except RuntimeError: clip_path = None if clip_path is None: direct_clip_path = str(clip.get("clipPath", "")).strip() if direct_clip_path and pathlib.Path(direct_clip_path).exists(): clip_path = pathlib.Path(direct_clip_path) if clip_path is None: clip_path = materialize_clip_from_source(clip, output_dir, archive_sources) if clip_path is None or not clip_path.exists(): raise RuntimeError(f"无法获取真实输入文件: {clip_name}") annotation = annotate_video_clip(clip_path, frame_interval) annotation_filename = f"{pathlib.Path(clip_name).stem}_annotation.json" annotation_payload = { "clipId": clip.get("clipId", pathlib.Path(clip_name).stem), "sourceVideo": clip.get("sourceVideo", ""), **annotation, } items.append({**clip, "annotation": annotation_payload, "annotationFile": annotation_filename}) write_json_artifact( run_id, node_id, output_dir / "annotations", annotation_filename, annotation_payload, output=output_binding(node_data), ) manifest = { "datasetType": "x-platform-annotations", "annotationCount": len(items), "items": items, } artifacts = [write_json_artifact(run_id, node_id, output_dir, "annotations-manifest.json", manifest, output=output_binding(node_data))] artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "annotations", node_data=node_data, output=output_binding(node_data))) return artifacts def builtin_result_package( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: payload = first_payload_with_keys(inputs, "annotationCount", "items") if payload is None: payload = first_payload_with_keys(inputs, "clipCount", "items") if payload is None: raise RuntimeError("成品打包需要上游标注或目标裁剪结果输入") clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")} archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {} packages = [] for item in payload.get("items", []): if not isinstance(item, dict): continue clip_name = str(item.get("clipName", "")).strip() clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem package_name = f"{clip_id}_package" package_dir = output_dir / "packages" / package_name package_dir.mkdir(parents=True, exist_ok=True) artifact = clip_artifacts.get(clip_name) clip_path: pathlib.Path | None = None if artifact is not None: try: clip_path = source_path_from_item( {"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")}, clip_name, output_dir, archive_sources, ) except RuntimeError: clip_path = None if clip_path is None: direct_clip_path = str(item.get("clipPath", "")).strip() if direct_clip_path and pathlib.Path(direct_clip_path).exists(): clip_path = pathlib.Path(direct_clip_path) if clip_path is None: clip_path = materialize_clip_from_source(item, output_dir, archive_sources) if clip_path is not None and clip_path.exists(): shutil.copy2(clip_path, package_dir / clip_name) quality_payload = { "clip_id": clip_id, "source_video": item.get("sourceVideo", ""), **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}), } annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {} if not annotation_payload and isinstance(item.get("detections"), list): annotation_payload = { "type": "target_detections", "detectionMode": item.get("detectionMode", payload.get("detectionMode", "")), "targetLabels": item.get("targetLabels", payload.get("targetLabels", [])), "detections": item.get("detections", []), "detectionCount": item.get("detectionCount", 0), "maxConfidence": item.get("maxConfidence", 0), } config_payload = package_config_payload(item, payload) write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_quality.json", quality_payload, output=output_binding(node_data)) write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation_payload, output=output_binding(node_data)) write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_config.json", config_payload, output=output_binding(node_data)) package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()] packages.append( { "packageName": package_name, "clipId": clip_id, "clipName": clip_name, "sourceVideo": item.get("sourceVideo", ""), "clipDuration": item.get("duration", 0), "duration": item.get("duration", 0), "startSecond": item.get("startSecond"), "endSecond": item.get("endSecond"), "collectionId": item.get("collectionId", ""), "meta": item.get("meta", {}), "source": item.get("source", {}) if isinstance(item.get("source"), dict) else {}, "fileCount": len(package_files), "pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1"], "storagePath": str(package_dir), "quality": quality_payload, "annotation": annotation_payload, } ) manifest = { "datasetType": "x-platform-packages", "packageCount": len(packages), "items": packages, } artifacts = [write_json_artifact(run_id, node_id, output_dir, "packages-manifest.json", manifest, output=output_binding(node_data))] artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "packages", node_data=node_data, output=output_binding(node_data))) return artifacts def builtin_audit_sampling( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) payload = first_payload_with_keys(inputs, "packageCount", "items") if payload is None: raise RuntimeError("抽检采样需要上游成品数据输入") rules = normalize_sampling_rules(parse_json_param(params, "RULES_JSON", [], list)) sampled_items = [] summaries = [] for rule in rules: rule_id = rule["id"] field_path = rule["field"] operator = rule["operator"] value = rule.get("value") percent = rule["samplePercent"] matched = [] sampled = [] for item in payload.get("items", []): if not isinstance(item, dict): continue if not compare_values(get_field_value(item, field_path), operator, value): continue matched.append(item) package_name = str(item.get("packageName", "")).strip() or str(item.get("clipId", "")).strip() if stable_sample_hit(f"{rule_id}:{package_name}", percent): sampled.append(item) sampled_items.append({**item, "sampleRuleId": rule_id}) summaries.append( { "id": rule_id, "field": field_path, "operator": operator, "value": value, "samplePercent": percent, "matchedCount": len(matched), "sampledCount": len(sampled), } ) manifest = { "datasetType": "x-platform-audit-samples", "rules": summaries, "sampledCount": len(sampled_items), "items": sampled_items, } return [write_json_artifact(run_id, node_id, output_dir, "audit-samples.json", manifest, output=output_binding(node_data))] def builtin_condition_router( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) field_path = params.get("FIELD_PATH", "").strip() operator = normalize_comparison_operator(params.get("OPERATOR", "eq")) expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", "")) rows = collect_items(payloads_from_inputs(inputs)) matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)] unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)] outputs = node_data.get("outputs", []) or [ {"id": "matched", "name": "matched", "type": "json"}, {"id": "unmatched", "name": "unmatched", "type": "json"}, ] return [ write_json_artifact( run_id, node_id, output_dir, f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json", {"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)}, str(outputs[0].get("type", "json")), output=outputs[0], ), write_json_artifact( run_id, node_id, output_dir, f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json", {"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)}, str(outputs[1].get("type", "json")), output=outputs[1], ), ] def builtin_lerobot_summary( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) source_path = resolve_source_path(params, inputs) if source_path is not None: root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent) payload = summarize_lerobot_root(root) payload["sourcePath"] = str(source_path) elif inputs: payload = summarize_lerobot_from_inputs(inputs) payload["sourcePath"] = "" else: raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录") payload["datasetType"] = "lerobot" return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))] def builtin_rosbag_summary( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) source_path = resolve_source_path(params, inputs) if source_path is not None: if source_path.is_dir(): payload = { "datasetType": "rosbag2", "sourcePath": str(source_path), "summary": summarize_directory_tree(source_path), } metadata_yaml = source_path / "metadata.yaml" if metadata_yaml.exists(): payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"] db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()] if db3_files: payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"] return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))] payload = summarize_rosbag_file(source_path) payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag") payload["sourcePath"] = str(source_path) else: for artifact in inputs: preview = artifact_preview_payload(artifact) or {} if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}: payload = { "datasetType": preview.get("format", "rosbag"), "sourcePath": str(artifact.get("path", "")), "preview": preview, } return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))] raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件") return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))] def builtin_generic( run_id: str, node_id: str, output_dir: pathlib.Path, node_label: str, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: payloads = [load_artifact_payload(item) for item in inputs] summary = { "nodeLabel": node_label, "inputCount": len(inputs), "inputs": [summarize_json_like(item) for item in payloads], "config": { "bucket": node_data.get("bucket", ""), "prefix": node_data.get("prefix", ""), "pattern": node_data.get("pattern", ""), }, } outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}] artifacts: list[dict[str, Any]] = [] for index, output in enumerate(outputs): output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}" filename = f"{output_name}.json" payload = {**summary, "outputName": output_name} artifacts.append( write_json_artifact( run_id, node_id, output_dir, filename, payload, str(output.get("type", "json")), output=output, ) ) return artifacts def builtin_model_inference( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.model_inference.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_vla_model_train( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.vla_model_train.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_sim3d_asset_generate( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.sim3d_asset_generate.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_holobrain_data_prepare( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.holobrain_data_prepare.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_holobrain_train( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.holobrain_train.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_holobrain_export( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.holobrain_export.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_holobrain_robotwin_eval( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.holobrain_robotwin_eval.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_holobrain_inference_test( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.holobrain_inference_test.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_vla_model_inference( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.vla_model_inference.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_vla_model_evaluate( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.vla_model_evaluate.operator', run_id, node_id, output_dir, node_data, inputs) def builtin_shell_command( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: return run_operator_module('backend.operators.builtin.shell_command.operator', run_id, node_id, output_dir, node_data, inputs) def execute_command( run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], command: str, ) -> list[dict[str, Any]]: validate_command_policy(command) completed = subprocess.run( command, cwd=output_dir, shell=True, text=True, capture_output=True, env=os.environ.copy(), check=False, ) for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)): for line in content.splitlines(): if line.strip(): log(f"[operator-runner][command][{stream_name}] {line}") if completed.returncode != 0: raise RuntimeError(f"command failed with exit code {completed.returncode}") artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data) if artifacts: return artifacts payload = {"stdout": completed.stdout, "stderr": completed.stderr} return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))] OPERATOR_HELPERS = { "TABULAR_FILE_SUFFIXES": TABULAR_FILE_SUFFIXES, "DEFAULT_MAX_HAND_OCCLUSION": DEFAULT_MAX_HAND_OCCLUSION, "DEFAULT_MAX_SHAKE": DEFAULT_MAX_SHAKE, "DEFAULT_MIN_CLARITY": DEFAULT_MIN_CLARITY, "RESERVED_MONGO_COLLECTIONS": RESERVED_MONGO_COLLECTIONS, "VALID_SOURCE_DATA_SOURCES": VALID_SOURCE_DATA_SOURCES, "VIDEO_FILE_SUFFIXES": VIDEO_FILE_SUFFIXES, "apply_incremental_source_filter": apply_incremental_source_filter, "artifact_sha256": artifact_sha256, "analyze_video_quality_metrics": analyze_video_quality_metrics, "annotate_video_clip": annotate_video_clip, "artifact_preview_payload": artifact_preview_payload, "bos_key_join": bos_key_join, "bos_prefix": bos_prefix, "build_bos_client": build_bos_client, "build_mongo_client": build_mongo_client, "build_artifact": build_artifact, "build_virtual_artifact": build_virtual_artifact, "canonical_bos_path": canonical_bos_path, "clean_metadata": clean_metadata, "collect_items": collect_items, "collect_output_artifacts": collect_output_artifacts, "common_input_root": common_input_root, "compare_values": compare_values, "filter_rows": filter_rows, "date_in_range": date_in_range, "documents_from_inputs": documents_from_inputs, "execute_command": execute_command, "extract_preview_data": extract_preview_data, "find_lerobot_root": find_lerobot_root, "ffmpeg_available": ffmpeg_available, "flatten_context": flatten_context, "first_payload_with_keys": first_payload_with_keys, "float_value": float_value, "get_field_value": get_field_value, "hyperparams": hyperparams, "incremental_directory_depth": incremental_directory_depth, "incremental_unit": incremental_unit, "infer_port_type": infer_port_type, "inspect_video_file": inspect_video_file, "json_compatible": json_compatible, "list_bos_entries": list_bos_entries, "list_source_entries": list_source_entries, "load_artifact_payload": load_artifact_payload, "load_tabular_rows": load_tabular_rows, "load_payload_from_file": load_payload_from_file, "load_xplatform_index_payload": load_xplatform_index_payload, "load_json_file": load_json_file, "log": log, "map_json_row": map_json_row, "materialize_clip_from_source": materialize_clip_from_source, "materialize_package_artifacts_from_manifest": materialize_package_artifacts_from_manifest, "materialized_upload_source": materialized_upload_source, "merge_dicts": merge_dicts, "normalize_json_payload": normalize_json_payload, "normalize_date_value": normalize_date_value, "normalize_comparison_operator": normalize_comparison_operator, "normalize_field_map": normalize_field_map, "normalize_sampling_rules": normalize_sampling_rules, "output_binding": output_binding, "package_config_payload": package_config_payload, "package_related_artifact_names": package_related_artifact_names, "parse_bool_param": parse_bool_param, "parse_bos_path": parse_bos_path, "parse_json_param": parse_json_param, "parse_positive_float_param": parse_positive_float_param, "parse_positive_int_param": parse_positive_int_param, "parse_unit_interval_param": parse_unit_interval_param, "payloads_from_inputs": payloads_from_inputs, "preview_payload_for_remote_entry": preview_payload_for_remote_entry, "preview_text_payload": preview_text_payload, "render_template_text": render_template_text, "resolve_source_path": resolve_source_path, "rows_from_payloads": rows_from_payloads, "source_path_from_item": source_path_from_item, "source_entry_artifact": source_entry_artifact, "source_ref_from_artifact": source_ref_from_artifact, "stable_sample_hit": stable_sample_hit, "summarize_directory_tree": summarize_directory_tree, "summarize_input_artifacts": summarize_input_artifacts, "summarize_lerobot_from_inputs": summarize_lerobot_from_inputs, "summarize_lerobot_root": summarize_lerobot_root, "summarize_rosbag_file": summarize_rosbag_file, "summarize_sqlite_payload": summarize_sqlite_payload, "text_value": text_value, "upload_relative_path": upload_relative_path, "write_csv_artifact": write_csv_artifact, "write_json_artifact": write_json_artifact, "write_text_artifact": write_text_artifact, "write_video_segment": write_video_segment, "write_video_segment_with_ffmpeg": write_video_segment_with_ffmpeg, "write_video_segments": write_video_segments, } BuiltinHandler = Any _OPERATOR_MODULE_CACHE: dict[str, Any] = {} _OPERATOR_MODULE_ALIASES: dict[str, str] = { f"{path.name}_operator": f"backend.operators.builtin.{path.name}.operator" for path in (PROJECT_ROOT / "backend" / "operators" / "builtin").iterdir() if path.is_dir() and (path / "operator.py").exists() } def load_operator_module(module_path: str) -> Any: module = _OPERATOR_MODULE_CACHE.get(module_path) if module is None: module = importlib.import_module(module_path) _OPERATOR_MODULE_CACHE[module_path] = module return module def __getattr__(name: str) -> Any: module_path = _OPERATOR_MODULE_ALIASES.get(name) if module_path: return load_operator_module(module_path) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def run_operator_module( module_path: str, run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: source_path = os.getenv("DROBOTICFLOW_OPERATOR_SOURCE", "").strip() if source_path: source = pathlib.Path(source_path) if not source.is_file(): raise RuntimeError(f"DROBOTICFLOW_OPERATOR_SOURCE does not exist: {source}") module_name = "droboticflow_split_operator_" + hashlib.sha1(str(source).encode("utf-8")).hexdigest() spec = importlib.util.spec_from_file_location(module_name, source) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load operator source: {source}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module.run(run_id, node_id, output_dir, node_data, inputs, OPERATOR_HELPERS) return load_operator_module(module_path).run(run_id, node_id, output_dir, node_data, inputs, OPERATOR_HELPERS) def operator_handler(module_path: str) -> BuiltinHandler: def run(run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]: return run_operator_module(module_path, run_id, node_id, output_dir, node_data, inputs) return run def builtin_generic_handler(run_id: str, node_id: str, output_dir: pathlib.Path, node_label: str, node_data: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]: return builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs) BUILTIN_HANDLER_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("bos_incremental_scan", ("bos 增量扫描", "bos incremental", "增量扫描")), ("human_review", ("人工审核", "人工验收", "任务审核", "human review")), ("mongodb_write", ("mongodb", "mongo 写入", "mongo write", "写入 mongo")), ("xplatform_index", ("x 平台数据集索引", "x-platform", "数据集索引", "dataset index")), ("collection_validate", ("采集目录校验", "目录校验", "validate collection")), ("video_quality", ("视频质检", "quality check", "quality")), ("dataset_filter", ("数据过滤", "dataset filter", "video filter")), ("video_slicing", ("视频切片", "video slicing", "slice")), ("yolo_video_target_clip", ("yolo 视频目标检测裁剪", "yolo target clip", "yolo video detection")), ("video_target_clip", ("视频目标检测裁剪", "目标检测裁剪", "视频目标检测", "target clip", "object detection clip", "video object detection")), ("video_annotation", ("视频标注", "annotation", "annotate")), ("result_package", ("成品打包", "package output", "package result")), ("audit_sampling", ("抽检采样", "audit sample", "sampling")), ("hf_dataset_probe", ("hf 数据集探测", "huggingface dataset", "hf dataset", "dataset viewer")), ("agibot_world2026_validate", ("agibotworld2026", "agibot world", "agibotworld", "智元 world", "智元数据集")), ("bridgedata_tfds_manifest_validate", ("bridgedata", "bridge data", "tfds-rlds", "tfds rlds 清单")), ("rlds_episode_summary", ("rlds", "tfds", "open x-embodiment", "轨迹概览")), ("droid_raw_episode_validate", ("droid raw", "droid episode", "droid trajectory", "droid raw episode", "droid 数据校验")), ("libero_hdf5_task_validate", ("libero hdf5", "libero 任务", "libero 数据集", "libero benchmark")), ("robocasa_hdf5_demonstration_validate", ("robocasa hdf5", "robocasa 示教", "robocasa 数据集", "robocasa demonstration")), ("furniturebench_pkl_demonstration_validate", ("furniturebench", "furniture bench", "家具装配", "furniture pkl")), ("roboturk_real_hdf5_validate", ("roboturk real", "roboturk hdf5", "roboturk aligned", "roboturk 真实", "robo turk")), ("roboset_hdf5_validate", ("roboset hdf5", "roboset 数据", "robo set", "robohive dataset", "robohive hdf5")), ("rh20t_scene_validate", ("rh20t", "rh20t 场景", "rh20t scene", "real human 20t")), ("robomind_hdf5_quality_validate", ("robomind hdf5", "robomind 质量", "robomind 数据", "multi-embodiment intelligence")), ("aloha_hdf5_validate", ("aloha hdf5", "aloha 示教", "act hdf5", "aloha 数据校验")), ("robomimic_hdf5_validate", ("robomimic hdf5", "robomimic 示教", "robomimic 数据校验", "robomimic dataset")), ("hdf5_demonstration_summary", ("hdf5 示教", "hdf5 demo", "robomimic", "droid hdf5")), ("holobrain_data_prepare", ("holobrain 数据准备", "holobrain data prepare", "robotwin2.0 数据准备", "robotwin 数据准备")), ("holobrain_train", ("holobrain 完整训练", "holobrain train", "holobrain 训练")), ("holobrain_export", ("holobrain 模型导出", "holobrain export")), ("holobrain_robotwin_eval", ("holobrain robotwin 评测", "holobrain robotwin eval", "robotwin 评测")), ("holobrain_inference_test", ("holobrain 模型测试", "holobrain inference test", "holobrain 推理测试")), ("openvla_finetune", ("openvla 微调", "openvla finetune", "openvla 训练")), ("openvla_inference", ("openvla 推理", "openvla inference")), ("vla_sample_manifest_validate", ("vla 样本", "vla manifest", "视觉语言动作样本", "droid manifest")), ("rlbench_dataset_validate", ("rlbench", "rlbench 数据集", "rlbench dataset")), ("maniskill_trajectory_validate", ("maniskill", "maniskill 轨迹", "mani skill")), ("calvin_episode_validate", ("calvin episode", "calvin 数据", "calvin episode 校验")), ("rtx_action_schema_validate", ("rt-x 动作", "rtx action", "open x action", "动作空间schema")), ("embodiment_metadata_validate", ("embodiment 元信息", "多机器人embodiment", "robot embodiment", "具身元信息")), ("force_torque_sensor_validate", ("力扭矩", "force torque", "wrench", "ft sensor", "力/扭矩")), ("lerobot_video_alignment_validate", ("lerobot 视频对齐", "lerobot video", "video alignment", "视频对齐校验")), ("lerobot_v3_episode_index_validate", ("lerobot v3 episode", "episode索引", "episode index", "data path template")), ("lerobot_dataset_validate", ("lerobot 数据集校验", "lerobot validate", "lerobot validation")), ("lerobot_metadata_consistency_validate", ("lerobot 元数据", "lerobot metadata", "metadata consistency")), ("trajectory_integrity_validate", ("轨迹多模态完整性校验", "trajectory integrity", "multimodal integrity")), ("action_state_quality_validate", ("动作状态数值", "action state quality", "action/state", "状态数值")), ("episode_boundary_validate", ("episode 边界", "episode boundary", "终止标志", "terminal flag")), ("joint_limit_validate", ("关节限位", "joint limit", "urdf limit", "qpos limit")), ("dataset_split_leakage_validate", ("切分泄漏", "split leakage", "leakage validation", "数据泄漏")), ("dataset_split_manifest", ("训练切分清单", "dataset split", "split manifest")), ("media_reference_validate", ("媒体引用", "media reference", "media readability", "引用可读")), ("dataset_schema_contract_validate", ("schema 合约", "schema contract", "数据集schema", "schema 校验")), ("mcap_topic_summary", ("mcap 主题概览", "mcap topic", "mcap summary")), ("tfrecord_structural_validate", ("tfrecord 结构校验", "tf record", "tfrecord validate")), ("dataset_provenance_audit", ("数据集来源许可审计", "dataset provenance", "license audit", "来源许可")), ("webdataset_shard_summary", ("webdataset 分片", "webdataset shard", "tar shard", "分片概览")), ("diffusion_policy_train", ("diffusion policy 训练", "diffusion policy train", "扩散策略训练")), ("umi_zarr_dataset_validate", ("umi zarr", "umi 数据", "universal manipulation interface", "diffusion policy zarr", "umi dataset")), ("zarr_trajectory_summary", ("zarr 轨迹", "zarr trajectory", "zarr array", "数组轨迹")), ("camera_calibration_validate", ("相机标定校验", "camera calibration", "intrinsics", "extrinsics")), ("tf_transform_tree_validate", ("tf 坐标树", "transform tree", "tf tree", "坐标树校验", "frametree")), ("ros_joint_state_validate", ("ros jointstate", "jointstate", "joint state", "joint_states", "ros 关节状态")), ("graspnet_dataset_validate", ("graspnet", "graspnet-1billion", "抓取数据集", "6d grasp", "general object grasping")), ("bop_dataset_validate", ("bop 6d", "bop 数据集", "bop dataset", "6d姿态", "6d pose", "object pose benchmark")), ("dexycb_dataset_validate", ("dexycb 数据集", "dex-ycb", "dexycb", "ycb hand", "hand grasping object")), ("ho3d_dataset_validate", ("ho3d 数据集", "ho-3d", "ho3d", "h2o3d", "h2o-3d", "honnotate")), ("wilor_hand_pose", ("wilor", "wilor 手部", "手部姿态估计", "3d hand pose", "hand pose estimation")), ("head_pose", ("headpose", "head pose", "头部姿态", "头部姿态估计")), ("delivery_options", ("交付选项透传", "delivery options")), ("hand_removal", ("hand removal", "视频去手", "手部移除", "去除手臂", "propainter")), ("hand_object_pose_validate", ("手-物体姿态", "手物体姿态", "hand object pose", "dexycb", "ho3d")), ("partnet_mobility_asset_validate", ("partnet-mobility", "partnet mobility", "sapien asset", "sapien 资产", "mobility.urdf", "可交互物体")), ("mjcf_actuator_validate", ("mjcf 执行器", "mjcf actuator", "mujoco actuator", "执行器校验", "ctrlrange")), ("robot_model_asset_validate", ("机器人模型资产校验", "robot model asset", "urdf", "mjcf")), ("pointcloud_summary", ("点云数据概览", "point cloud", "pointcloud", "pcd", "ply")), ("sensor_timestamp_sync_validate", ("多传感器时间同步", "timestamp sync", "sensor sync", "时间同步")), ("parquet_table_summary", ("parquet 表摘要", "parquet summary", "parquet table", "parquet")), ("coco_annotation_validate", ("coco 标注", "coco annotation", "coco validate", "coco")), ("yolo_embodied_detection_train", ("yolo 具身目标检测训练", "yolo detection train", "yolo 训练")), ("yolo_video_target_clip", ("yolo 视频目标检测裁剪", "yolo target clip", "yolo video detection")), ("yolo_detection_validate", ("yolo 检测", "yolo label", "yolo 标注", "yolo")), ("rgbd_frame_pair_validate", ("rgb-d", "rgbd", "rgb-d 帧", "深度帧", "帧对齐")), ("segmentation_mask_validate", ("分割 mask", "segmentation mask", "semantic mask", "mask 校验")), ("language_instruction_validate", ("语言指令", "language instruction", "instruction validate", "vla 指令")), ("lerobot", ("lerobot",)), ("rosbag", ("rosbag", "mcap", "ros bag")), ("csv_to_json", ("csv 转 json", "csv转json", "csv to json", "tsv 转 json", "tsv转json", "tsv to json")), ("json_to_csv", ("json 转 csv", "json转csv", "json to csv")), ("json_extract", ("字段提取", "json提取", "json 提取", "extract")), ("json_filter", ("json过滤", "json 过滤", "filter")), ("json_map", ("json映射", "json 映射", "rename", "映射重命名")), ("text_template", ("模板渲染", "template")), ("condition_router", ("条件分流", "conditional", "router", "route")), ("directory_scan", ("目录读取", "目录扫描", "directory", "browse", "browse dir")), ("merge", ("merge", "合并")), ("intersection", ("intersect", "交集")), ("distribution", ("distribution", "分配")), ("upload", ("upload", "export", "上传", "导出")), ("act_policy_train", ("act 策略训练", "act policy train", "act training")), ("vla_model_train", ("vla 模型训练", "vla training", "视觉语言动作训练", "policy training")), ("sim3d_asset_generate", ("3d 资产仿真生成", "3d资产仿真生成", "仿真资产生成", "simulation asset", "sim3d asset")), ("vla_model_inference", ("vla 模型推理", "vla inference", "视觉语言动作推理", "policy inference")), ("vla_model_evaluate", ("vla 模型评测", "vla 评测", "vla evaluation", "policy evaluation")), ("model_inference", ("模型推理", "model inference", "inference")), ("shell_command", ("shell 命令", "shell command")), ("source", ("source", "scanner", "oss", "scan", "扫描", "数据源")), ) BUILTIN_HANDLERS: dict[str, BuiltinHandler] = { "source": operator_handler('backend.operators.builtin.real_source.operator'), "act_policy_train": operator_handler('backend.operators.builtin.act_policy_train.operator'), "bos_incremental_scan": operator_handler('backend.operators.builtin.bos_incremental_scan.operator'), "merge": operator_handler('backend.operators.builtin.merge.operator'), "intersection": builtin_intersection, "distribution": builtin_distribution, "json_extract": operator_handler('backend.operators.builtin.json_field_extract.operator'), "json_filter": operator_handler('backend.operators.builtin.json_filter.operator'), "json_map": operator_handler('backend.operators.builtin.json_map_rename.operator'), "mongodb_write": operator_handler('backend.operators.builtin.mongodb_write.operator'), "text_template": operator_handler('backend.operators.builtin.template_render.operator'), "csv_to_json": operator_handler('backend.operators.builtin.csv_to_json.operator'), "json_to_csv": operator_handler('backend.operators.builtin.json_to_csv.operator'), "directory_scan": operator_handler('backend.operators.builtin.directory_scan.operator'), "diffusion_policy_train": operator_handler('backend.operators.builtin.diffusion_policy_train.operator'), "xplatform_index": operator_handler('backend.operators.builtin.xplatform_dataset_index.operator'), "collection_validate": operator_handler('backend.operators.builtin.xplatform_dataset_validate.operator'), "video_quality": operator_handler('backend.operators.builtin.video_quality.operator'), "dataset_filter": operator_handler('backend.operators.builtin.video_filter.operator'), "video_slicing": operator_handler('backend.operators.builtin.video_slice.operator'), "video_face_blur": operator_handler('backend.operators.builtin.video_face_blur.operator'), "qwen_behavior_rough_cut": operator_handler('backend.operators.builtin.qwen_behavior_rough_cut.operator'), "qwen_temporal_localization": operator_handler('backend.operators.builtin.qwen_temporal_localization.operator'), "wilor_valid_event_detection": operator_handler('backend.operators.builtin.wilor_valid_event_detection.operator'), "ms_swift_dataset_export": operator_handler('backend.operators.builtin.ms_swift_dataset_export.operator'), "qwen_label_crop_classification": operator_handler('backend.operators.builtin.qwen_label_crop_classification.operator'), "video_target_clip": operator_handler('backend.operators.builtin.video_target_clip.operator'), "video_annotation": operator_handler('backend.operators.builtin.video_annotate.operator'), "annotation_gate": operator_handler('backend.operators.builtin.annotation_gate.operator'), "orbbec_data_normalize": operator_handler('backend.operators.builtin.orbbec_data_normalize.operator'), "orbbec_left_undistort": operator_handler('backend.operators.builtin.orbbec_left_undistort.operator'), "semantic_prelabel": operator_handler('backend.operators.builtin.semantic_prelabel.operator'), "result_package": operator_handler('backend.operators.builtin.package.operator'), "audit_sampling": operator_handler('backend.operators.builtin.audit_sample.operator'), "hf_dataset_probe": operator_handler('backend.operators.builtin.hf_dataset_probe.operator'), "agibot_world2026_validate": operator_handler('backend.operators.builtin.agibot_world2026_validate.operator'), "rlds_episode_summary": operator_handler('backend.operators.builtin.rlds_episode_summary.operator'), "bridgedata_tfds_manifest_validate": operator_handler('backend.operators.builtin.bridgedata_tfds_manifest_validate.operator'), "droid_raw_episode_validate": operator_handler('backend.operators.builtin.droid_raw_episode_validate.operator'), "libero_hdf5_task_validate": operator_handler('backend.operators.builtin.libero_hdf5_task_validate.operator'), "hdf5_demonstration_summary": operator_handler('backend.operators.builtin.hdf5_demonstration_summary.operator'), "aloha_hdf5_validate": operator_handler('backend.operators.builtin.aloha_hdf5_validate.operator'), "robomimic_hdf5_validate": operator_handler('backend.operators.builtin.robomimic_hdf5_validate.operator'), "robocasa_hdf5_demonstration_validate": operator_handler('backend.operators.builtin.robocasa_hdf5_demonstration_validate.operator'), "furniturebench_pkl_demonstration_validate": operator_handler('backend.operators.builtin.furniturebench_pkl_demonstration_validate.operator'), "roboturk_real_hdf5_validate": operator_handler('backend.operators.builtin.roboturk_real_hdf5_validate.operator'), "roboset_hdf5_validate": operator_handler('backend.operators.builtin.roboset_hdf5_validate.operator'), "rh20t_scene_validate": operator_handler('backend.operators.builtin.rh20t_scene_validate.operator'), "robomind_hdf5_quality_validate": operator_handler('backend.operators.builtin.robomind_hdf5_quality_validate.operator'), "vla_sample_manifest_validate": operator_handler('backend.operators.builtin.vla_sample_manifest_validate.operator'), "rlbench_dataset_validate": operator_handler('backend.operators.builtin.rlbench_dataset_validate.operator'), "maniskill_trajectory_validate": operator_handler('backend.operators.builtin.maniskill_trajectory_validate.operator'), "calvin_episode_validate": operator_handler('backend.operators.builtin.calvin_episode_validate.operator'), "rtx_action_schema_validate": operator_handler('backend.operators.builtin.rtx_action_schema_validate.operator'), "embodiment_metadata_validate": operator_handler('backend.operators.builtin.embodiment_metadata_validate.operator'), "force_torque_sensor_validate": operator_handler('backend.operators.builtin.force_torque_sensor_validate.operator'), "lerobot_video_alignment_validate": operator_handler('backend.operators.builtin.lerobot_video_alignment_validate.operator'), "lerobot_v3_episode_index_validate": operator_handler('backend.operators.builtin.lerobot_v3_episode_index_validate.operator'), "lerobot_dataset_validate": operator_handler('backend.operators.builtin.lerobot_dataset_validate.operator'), "lerobot_metadata_consistency_validate": operator_handler('backend.operators.builtin.lerobot_metadata_consistency_validate.operator'), "trajectory_integrity_validate": operator_handler('backend.operators.builtin.trajectory_integrity_validate.operator'), "action_state_quality_validate": operator_handler('backend.operators.builtin.action_state_quality_validate.operator'), "episode_boundary_validate": operator_handler('backend.operators.builtin.episode_boundary_validate.operator'), "joint_limit_validate": operator_handler('backend.operators.builtin.joint_limit_validate.operator'), "dataset_split_leakage_validate": operator_handler('backend.operators.builtin.dataset_split_leakage_validate.operator'), "dataset_split_manifest": operator_handler('backend.operators.builtin.dataset_split_manifest.operator'), "media_reference_validate": operator_handler('backend.operators.builtin.media_reference_validate.operator'), "dataset_schema_contract_validate": operator_handler('backend.operators.builtin.dataset_schema_contract_validate.operator'), "mcap_topic_summary": operator_handler('backend.operators.builtin.mcap_topic_summary.operator'), "tfrecord_structural_validate": operator_handler('backend.operators.builtin.tfrecord_structural_validate.operator'), "dataset_provenance_audit": operator_handler('backend.operators.builtin.dataset_provenance_audit.operator'), "graspnet_dataset_validate": operator_handler('backend.operators.builtin.graspnet_dataset_validate.operator'), "bop_dataset_validate": operator_handler('backend.operators.builtin.bop_dataset_validate.operator'), "dexycb_dataset_validate": operator_handler('backend.operators.builtin.dexycb_dataset_validate.operator'), "ho3d_dataset_validate": operator_handler('backend.operators.builtin.ho3d_dataset_validate.operator'), "wilor_hand_pose": operator_handler('backend.operators.builtin.wilor_hand_pose.operator'), "head_pose": operator_handler('backend.operators.builtin.head_pose.operator'), "delivery_options": operator_handler('backend.operators.builtin.delivery_options.operator'), "hand_removal": operator_handler('backend.operators.builtin.hand_removal.operator'), "webdataset_shard_summary": operator_handler('backend.operators.builtin.webdataset_shard_summary.operator'), "umi_zarr_dataset_validate": operator_handler('backend.operators.builtin.umi_zarr_dataset_validate.operator'), "zarr_trajectory_summary": operator_handler('backend.operators.builtin.zarr_trajectory_summary.operator'), "camera_calibration_validate": operator_handler('backend.operators.builtin.camera_calibration_validate.operator'), "tf_transform_tree_validate": operator_handler('backend.operators.builtin.tf_transform_tree_validate.operator'), "ros_joint_state_validate": operator_handler('backend.operators.builtin.ros_joint_state_validate.operator'), "hand_object_pose_validate": operator_handler('backend.operators.builtin.hand_object_pose_validate.operator'), "partnet_mobility_asset_validate": operator_handler('backend.operators.builtin.partnet_mobility_asset_validate.operator'), "mjcf_actuator_validate": operator_handler('backend.operators.builtin.mjcf_actuator_validate.operator'), "robot_model_asset_validate": operator_handler('backend.operators.builtin.robot_model_asset_validate.operator'), "pointcloud_summary": operator_handler('backend.operators.builtin.pointcloud_summary.operator'), "sensor_timestamp_sync_validate": operator_handler('backend.operators.builtin.sensor_timestamp_sync_validate.operator'), "parquet_table_summary": operator_handler('backend.operators.builtin.parquet_table_summary.operator'), "coco_annotation_validate": operator_handler('backend.operators.builtin.coco_annotation_validate.operator'), "yolo_detection_validate": operator_handler('backend.operators.builtin.yolo_detection_validate.operator'), "rgbd_frame_pair_validate": operator_handler('backend.operators.builtin.rgbd_frame_pair_validate.operator'), "segmentation_mask_validate": operator_handler('backend.operators.builtin.segmentation_mask_validate.operator'), "language_instruction_validate": operator_handler('backend.operators.builtin.language_instruction_validate.operator'), "condition_router": operator_handler('backend.operators.builtin.conditional_route.operator'), "human_review": operator_handler('backend.operators.builtin.human_review.operator'), "builtin_human_review": operator_handler('backend.operators.builtin.human_review.operator'), "lerobot": operator_handler('backend.operators.builtin.lerobot_summary.operator'), "rosbag": operator_handler('backend.operators.builtin.rosbag_summary.operator'), "upload": operator_handler('backend.operators.builtin.upload.operator'), "holobrain_data_prepare": operator_handler('backend.operators.builtin.holobrain_data_prepare.operator'), "holobrain_train": operator_handler('backend.operators.builtin.holobrain_train.operator'), "holobrain_export": operator_handler('backend.operators.builtin.holobrain_export.operator'), "holobrain_robotwin_eval": operator_handler('backend.operators.builtin.holobrain_robotwin_eval.operator'), "holobrain_inference_test": operator_handler('backend.operators.builtin.holobrain_inference_test.operator'), "openvla_finetune": operator_handler('backend.operators.builtin.openvla_finetune.operator'), "openvla_inference": operator_handler('backend.operators.builtin.openvla_inference.operator'), "vla_model_train": operator_handler('backend.operators.builtin.vla_model_train.operator'), "sim3d_asset_generate": operator_handler('backend.operators.builtin.sim3d_asset_generate.operator'), "vla_model_inference": operator_handler('backend.operators.builtin.vla_model_inference.operator'), "vla_model_evaluate": operator_handler('backend.operators.builtin.vla_model_evaluate.operator'), "model_inference": builtin_model_inference, "yolo_embodied_detection_train": operator_handler('backend.operators.builtin.yolo_embodied_detection_train.operator'), "yolo_video_target_clip": operator_handler('backend.operators.builtin.yolo_video_target_clip.operator'), "shell_command": builtin_shell_command, "generic": builtin_generic_handler, } for _handler_name, _handler in tuple(BUILTIN_HANDLERS.items()): if _handler_name != "generic": BUILTIN_HANDLERS.setdefault(f"builtin_{_handler_name}", _handler) BUILTIN_HANDLERS.setdefault("builtin_lerobot_summary", BUILTIN_HANDLERS["lerobot"]) BUILTIN_HANDLERS.setdefault("builtin_rosbag_summary", BUILTIN_HANDLERS["rosbag"]) def select_handler(node_label: str) -> str: normalized = node_label.lower() for handler, tokens in BUILTIN_HANDLER_RULES: if any(token in normalized for token in tokens): return handler return "generic" def runtime_handler_override() -> str: handler = os.getenv("DROBOTICFLOW_OPERATOR_HANDLER", "").strip() if not handler: return "" if handler not in BUILTIN_HANDLERS: known = ", ".join(sorted(BUILTIN_HANDLERS)) raise RuntimeError(f"DROBOTICFLOW_OPERATOR_HANDLER={handler!r} is not registered; known handlers: {known}") return handler def main() -> int: run_id = os.getenv("RUN_ID", "").strip() workflow_id = os.getenv("WORKFLOW_ID", "").strip() node_id = os.getenv("NODE_ID", "").strip() node_label = os.getenv("NODE_LABEL", "").strip() or node_id output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id))) output_dir.mkdir(parents=True, exist_ok=True) node_data = load_json_env("NODE_DATA_JSON", {}) inputs = load_input_artifacts() input_bindings = load_input_bindings() started_at = now_iso() start_cpu, start_memory = resource_snapshot() sampler = ResourceSampler(start_cpu, start_memory, started_at) sampler.start() logs: list[dict[str, Any]] = [] def append_log(level: str, message: str) -> None: logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label}) log(f"[operator-runner][{level.lower()}] {message}") append_log( "INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir} input_bindings={len(input_bindings)}", ) append_log("INFO", "Preparing operator execution context") try: params = hyperparams(node_data) command = params.get("COMMAND") or params.get("SCRIPT") artifacts: list[dict[str, Any]] if command: append_log("INFO", "Executing explicit command") artifacts = execute_command(run_id, node_id, output_dir, node_data, command) else: handler = runtime_handler_override() or select_handler(node_label) append_log("INFO", f"Selected built-in handler: {handler}") builtin_handler = BUILTIN_HANDLERS.get(handler, BUILTIN_HANDLERS["generic"]) if handler == "generic": artifacts = builtin_handler(run_id, node_id, output_dir, node_label, node_data, inputs) else: artifacts = builtin_handler(run_id, node_id, output_dir, node_data, inputs) finally: ended_at = now_iso() resource_samples = sampler.stop(ended_at) append_log("INFO", f"Produced {len(artifacts)} artifact(s)") data_metrics = infer_data_metrics(node_data, artifacts, inputs) result = { "logs": logs, "artifacts": artifacts, "resourceSeries": resource_samples, } if data_metrics: result["dataMetrics"] = data_metrics stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip() if stage_manifest_path: write_json_file( stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics), ) else: write_json_file( default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics), ) result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip() if result_output_path: written_path = write_json_file(result_output_path, result) print(RESULT_PATH_PREFIX + str(written_path), flush=True) return 0 default_result_path = write_json_file(default_result_output_path(output_dir), result) print(RESULT_PATH_PREFIX + str(default_result_path), flush=True) return 0 def tempfile_dir(run_id: str, node_id: str) -> str: base = pathlib.Path("/tmp/droboticflow") return str(base / (run_id or "run") / (node_id or "node")) if __name__ == "__main__": try: sys.exit(main()) except Exception as error: # pragma: no cover log(f"[operator-runner][error] {error}") sys.exit(1)