2026-08-17 20:44:16 +08:00

5491 lines
235 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
from __future__ import annotations
import datetime as dt
import contextlib
import fnmatch
import hashlib
import importlib
import json
import mimetypes
import os
import pathlib
import base64
import csv
import math
import re
import shutil
import sqlite3
import subprocess
import sys
import tarfile
import time
import zipfile
from typing import Any
from urllib import error, request
from urllib.parse import urlparse
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
RUNNER_ROOT = pathlib.Path(__file__).resolve().parent
if str(RUNNER_ROOT) not in sys.path:
sys.path.insert(0, str(RUNNER_ROOT))
from resource_monitor import ResourceSampler, resource_snapshot
RESULT_PREFIX = "DROBOTICFLOW_RESULT="
RESULT_PATH_PREFIX = "DROBOTICFLOW_RESULT_PATH="
TEXT_FILE_SUFFIXES = {
".txt",
".log",
".md",
".yaml",
".yml",
".xml",
".html",
".htm",
}
TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"}
IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"}
ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"}
ARCHIVE_FILE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip")
XPLATFORM_VIDEO_META_FILES = ("video_meta.json", "video_metadata.json")
XPLATFORM_INLINE_JSON_FILES = {"meta.json", "intrinsics.json", *XPLATFORM_VIDEO_META_FILES}
PREVIEW_TEXT_LIMIT = 12_000
PREVIEW_TEXT_LINE_LIMIT = 240
PREVIEW_TABLE_ROW_LIMIT = 24
PREVIEW_FILE_LIST_LIMIT = 40
PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024
DATASET_PARTITION_SIZE = 500
QUALITY_SAMPLE_LIMIT = 48
QUALITY_ANALYSIS_MAX_DIMENSION = 640
QUALITY_SKIN_COMPONENT_MIN_RATIO = 0.002
DEFAULT_MIN_CLARITY = 0.8
DEFAULT_MAX_SHAKE = 0.1
DEFAULT_MAX_HAND_OCCLUSION = 0.1
TRUE_ENV_VALUES = {"1", "true", "yes", "on"}
FALSE_ENV_VALUES = {"0", "false", "no", "off"}
MAX_SAFE_INTEGER = 9_007_199_254_740_991
EXACT_POSITIVE_INT_RE = re.compile(r"^[1-9][0-9]*$")
INTEGERISH_RE = re.compile(r"^[+-]?[0-9]+$")
SUPPORTED_COMPARISON_OPERATORS = {"contains", "eq", "exists", "gt", "gte", "in", "lt", "lte", "missing", "ne"}
VALID_SOURCE_DATA_SOURCES = {"INLINE", "LOCAL", "BOS", "UPSTREAM"}
RESERVED_MONGO_COLLECTIONS = {
"projects",
"operators",
"workflows",
"templates",
"runs",
"sourceStates",
"auditTasks",
"samplingTasks",
"users",
"storageConnections",
"settings",
}
def env_bool(name: str, default: bool) -> bool:
value = os.getenv(name, "").strip().lower()
if value == "":
return default
if value in TRUE_ENV_VALUES:
return True
if value in FALSE_ENV_VALUES:
return False
raise RuntimeError(f"{name} must be one of true/false/1/0/yes/no/on/off")
def compile_policy_regex(name: str) -> re.Pattern[str] | None:
value = os.getenv(name, "").strip()
if not value:
return None
try:
return re.compile(value)
except re.error as error:
raise RuntimeError(f"{name} is not a valid regular expression: {error}") from error
def validate_command_policy(command: str) -> None:
if not env_bool("OPERATOR_ALLOW_SHELL_COMMANDS", False):
raise RuntimeError("explicit shell commands are disabled by OPERATOR_ALLOW_SHELL_COMMANDS")
denylist = compile_policy_regex("OPERATOR_COMMAND_DENYLIST_REGEX")
if denylist and denylist.search(command):
raise RuntimeError("explicit shell command was rejected by OPERATOR_COMMAND_DENYLIST_REGEX")
allowlist = compile_policy_regex("OPERATOR_COMMAND_ALLOWLIST_REGEX")
if allowlist and not allowlist.search(command):
raise RuntimeError("explicit shell command was not allowed by OPERATOR_COMMAND_ALLOWLIST_REGEX")
DATASET_PARTITION_KEYS = (
"items",
"entries",
"rows",
"collections",
"catalogCollections",
"catalogItems",
"catalogEntries",
)
PAYLOAD_FILE_READ_ATTEMPTS = 60
PAYLOAD_FILE_READ_DELAY_SECONDS = 1.0
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def log(message: str) -> None:
print(message, flush=True)
def load_json_env(name: str, default: Any) -> Any:
value = os.getenv(name, "").strip()
if not value:
return default
return json.loads(value)
def load_json_file(path: str, default: Any) -> Any:
raw = str(path or "").strip()
if not raw:
return default
candidate = pathlib.Path(raw)
if not candidate.exists():
return default
return json.loads(candidate.read_text(encoding="utf-8"))
def write_json_file(path: str | pathlib.Path, value: Any) -> pathlib.Path:
candidate = pathlib.Path(path)
candidate.parent.mkdir(parents=True, exist_ok=True)
candidate.write_text(json.dumps(json_compatible(value), ensure_ascii=False), encoding="utf-8")
return candidate
def text_value(value: Any) -> str:
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
return str(value or "")
def json_compatible(value: Any) -> Any:
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
if isinstance(value, dict):
return {str(key): json_compatible(item) for key, item in value.items()}
if isinstance(value, list):
return [json_compatible(item) for item in value]
if isinstance(value, tuple):
return [json_compatible(item) for item in value]
return value
def load_input_artifacts() -> list[dict[str, Any]]:
input_artifacts_path = os.getenv("INPUT_ARTIFACTS_PATH", "").strip()
if input_artifacts_path:
loaded = load_json_file(input_artifacts_path, [])
if isinstance(loaded, list):
return json_compatible(loaded)
loaded = load_json_env("INPUT_ARTIFACTS_JSON", [])
if isinstance(loaded, list):
return json_compatible(loaded)
return []
def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None:
for key in keys:
value = payload.get(key)
if isinstance(value, bool):
continue
if isinstance(value, (int, float)) and value >= 0:
return value
return None
def _metric_volume(payloads: list[dict[str, Any]]) -> dict[str, int | float]:
candidates: dict[str, list[int | float]] = {
"trajectoryCount": [],
"durationSeconds": [],
"bytes": [],
"fileCount": [],
}
for payload in payloads:
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
trajectory_count = _first_numeric(
payload,
("trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"),
)
if trajectory_count is None and data_ref:
trajectory_count = _first_numeric(data_ref, ("itemCount",))
duration_seconds = _first_numeric(payload, ("durationSeconds", "totalDurationSeconds", "totalDuration", "duration"))
total_bytes = _first_numeric(payload, ("totalBytes", "bytes", "sizeBytes"))
file_count = _first_numeric(payload, ("fileCount", "filesCount"))
if trajectory_count is not None:
candidates["trajectoryCount"].append(int(trajectory_count))
if duration_seconds is not None:
candidates["durationSeconds"].append(float(duration_seconds))
if total_bytes is not None:
candidates["bytes"].append(int(total_bytes))
if file_count is not None:
candidates["fileCount"].append(int(file_count))
return {key: max(values) for key, values in candidates.items() if values}
def _artifact_payloads(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [item["data"] for item in artifacts if isinstance(item.get("data"), dict)]
def _dataset_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
refs: list[dict[str, Any]] = []
seen: set[str] = set()
for artifact in artifacts:
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
if not data_ref:
continue
item = {
"name": text_value(artifact.get("name")),
"kind": text_value(data_ref.get("kind")),
"datasetType": text_value(data_ref.get("datasetType")),
"manifestPath": text_value(data_ref.get("manifestPath")),
"payloadPath": text_value(data_ref.get("payloadPath")),
"partitionDir": text_value(data_ref.get("partitionDir")),
"partitionCount": int(data_ref.get("partitionCount") or 0),
"itemCount": int(data_ref.get("itemCount") or 0),
"datasetId": text_value(data_ref.get("datasetId")),
"datasetVersionId": text_value(data_ref.get("datasetVersionId")),
"split": text_value(data_ref.get("split")),
"fingerprint": text_value(data_ref.get("fingerprint")),
}
key = "|".join(
text_value(item.get(name))
for name in ("datasetVersionId", "datasetId", "manifestPath", "payloadPath", "partitionDir", "split")
)
if key in seen:
continue
seen.add(key)
refs.append(item)
return refs
def infer_data_metrics(
node_data: dict[str, Any],
artifacts: list[dict[str, Any]],
inputs: list[dict[str, Any]],
) -> dict[str, Any] | None:
stage = text_value(node_data.get("businessStage")).strip().lower()
if stage not in {"collect", "process", "clean", "validate", "annotate", "audit", "train"}:
return None
if node_data.get("metricBoundary") is False:
return None
output_payloads = _artifact_payloads(artifacts)
for payload in output_payloads:
explicit = payload.get("dataMetrics")
if isinstance(explicit, dict):
normalized = json_compatible(explicit)
normalized.setdefault("schemaVersion", 1)
normalized.setdefault("stage", stage)
normalized.setdefault("quality", "exact")
normalized.setdefault("occurredAt", now_iso())
return normalized
input_payloads = _artifact_payloads(inputs)
input_volume = _metric_volume(input_payloads)
output_volume = _metric_volume(output_payloads)
metrics: dict[str, Any] = {
"schemaVersion": 1,
"stage": stage,
"input": input_volume,
"output": output_volume,
"datasetRefs": _dataset_refs(artifacts),
"quality": "derived",
"occurredAt": now_iso(),
}
if stage == "collect" and not input_volume:
metrics.pop("input")
if stage == "validate":
valid = any(payload.get("valid") is True for payload in output_payloads)
metrics["technicalValid"] = output_volume if valid else {}
if not valid and input_volume:
metrics["rejected"] = input_volume
if stage in {"annotate", "audit"}:
metrics["trainReady"] = output_volume
if stage == "clean" and input_volume and output_volume:
rejected: dict[str, int | float] = {}
for key in ("trajectoryCount", "durationSeconds", "bytes", "fileCount"):
if key in input_volume and key in output_volume:
rejected[key] = max(0, input_volume[key] - output_volume[key])
if rejected:
metrics["rejected"] = rejected
if stage == "train":
training_inputs: list[dict[str, Any]] = []
for ref in _dataset_refs(inputs):
training_inputs.append(
{
"datasetId": ref.get("datasetId", ""),
"datasetVersionId": ref.get("datasetVersionId", ""),
"name": ref.get("name", ""),
"split": ref.get("split", ""),
"volume": input_volume,
"manifestPath": ref.get("manifestPath", ""),
"partitionDir": ref.get("partitionDir", ""),
"external": not bool(ref.get("datasetVersionId")),
"quality": "derived",
}
)
metrics["trainingInputs"] = training_inputs
return metrics
def stage_manifest_payload(
run_id: str,
workflow_id: str,
node_id: str,
node_label: str,
output_dir: pathlib.Path,
artifacts: list[dict[str, Any]],
data_metrics: dict[str, Any] | None = None,
) -> dict[str, Any]:
manifest_artifacts: list[dict[str, Any]] = []
data_refs: list[dict[str, Any]] = []
for artifact in artifacts:
item = {
"id": text_value(artifact.get("id")),
"name": text_value(artifact.get("name")),
"type": text_value(artifact.get("type")),
"path": text_value(artifact.get("path")),
"size": text_value(artifact.get("size")),
}
if text_value(artifact.get("portId")):
item["portId"] = text_value(artifact.get("portId"))
if text_value(artifact.get("portName")):
item["portName"] = text_value(artifact.get("portName"))
if isinstance(artifact.get("metadata"), dict) and artifact["metadata"]:
item["metadata"] = json_compatible(artifact["metadata"])
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
if data_ref:
item["dataRef"] = json_compatible(data_ref)
data_refs.append(
{
"artifactId": item["id"],
"artifactName": item["name"],
"portId": item.get("portId", ""),
**json_compatible(data_ref),
}
)
manifest_artifacts.append(item)
result = {
"runId": run_id,
"workflowId": workflow_id,
"nodeId": node_id,
"nodeLabel": node_label,
"outputDir": str(output_dir),
"artifactCount": len(manifest_artifacts),
"artifacts": manifest_artifacts,
"dataRefs": data_refs,
"primaryDataRefs": data_refs,
"generatedAt": now_iso(),
}
if data_metrics:
result["dataMetrics"] = json_compatible(data_metrics)
return result
def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path:
return output_dir.parent / "result.json"
def default_stage_manifest_path(output_dir: pathlib.Path) -> pathlib.Path:
return output_dir.parent / "stage-manifest.json"
def normalize_json_payload(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return json_compatible(value)
if isinstance(value, list):
return {"items": json_compatible(value), "itemCount": len(value)}
return {"value": json_compatible(value)}
def bos_prefix(value: str) -> str:
return str(value or "").strip().strip("/")
def canonical_bos_path(bucket: str, key: str) -> str:
normalized_bucket = str(bucket or "").strip()
normalized_key = str(key or "").strip().lstrip("/")
if not normalized_bucket:
return ""
if not normalized_key:
return f"bos://{normalized_bucket}"
return f"bos://{normalized_bucket}/{normalized_key}"
def parse_bos_path(value: str) -> tuple[str, str]:
raw = str(value or "").strip()
if not raw.startswith("bos://"):
return "", ""
bucket_and_key = raw[len("bos://") :]
if not bucket_and_key:
return "", ""
bucket, _, key = bucket_and_key.partition("/")
return bucket.strip(), bos_prefix(key)
def is_xplatform_video_meta_name(value: str) -> bool:
return str(value or "").strip().lower() in XPLATFORM_VIDEO_META_FILES
def load_collection_video_meta(collection_dir: pathlib.Path) -> tuple[dict[str, dict[str, Any]], str]:
for candidate in XPLATFORM_VIDEO_META_FILES:
payload = optional_json_file(collection_dir / candidate)
if payload is not None:
return parse_video_meta_rows(payload), candidate
return {}, ""
def sampled_frame_positions(frame_count: int, sample_limit: int = QUALITY_SAMPLE_LIMIT) -> list[int]:
if frame_count <= 0:
return []
if frame_count <= sample_limit:
return list(range(frame_count))
last_frame = max(frame_count - 1, 0)
positions = {
min(last_frame, max(0, int(round(index * last_frame / max(sample_limit - 1, 1)))))
for index in range(sample_limit)
}
return sorted(positions)
def resize_quality_frame(frame: Any, max_dimension: int = QUALITY_ANALYSIS_MAX_DIMENSION) -> Any:
try:
height, width = frame.shape[:2]
except Exception:
return frame
longest_edge = max(width, height)
if longest_edge <= 0 or longest_edge <= max_dimension:
return frame
scale = max_dimension / float(longest_edge)
target_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
cv2, _ = require_cv2()
return cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA)
def build_skin_mask(frame: Any) -> Any:
cv2, np = require_cv2()
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
ycrcb_mask = cv2.inRange(
ycrcb,
np.array((0, 133, 77), dtype=np.uint8),
np.array((255, 173, 127), dtype=np.uint8),
)
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
hsv_mask = cv2.inRange(
hsv,
np.array((0, 20, 40), dtype=np.uint8),
np.array((25, 220, 255), dtype=np.uint8),
)
return cv2.bitwise_and(ycrcb_mask, hsv_mask)
def largest_skin_component_ratio(mask: Any, min_component_ratio: float = QUALITY_SKIN_COMPONENT_MIN_RATIO) -> float:
cv2, np = require_cv2()
if mask is None:
return 0.0
total_area = int(mask.shape[0] * mask.shape[1]) if len(mask.shape) >= 2 else 0
if total_area <= 0:
return 0.0
kernel = np.ones((5, 5), dtype=np.uint8)
cleaned = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_CLOSE, kernel)
component_threshold = max(1, int(total_area * max(min_component_ratio, 0.0)))
count, _, stats, _ = cv2.connectedComponentsWithStats(cleaned)
largest_area = 0
for index in range(1, count):
area = int(stats[index, cv2.CC_STAT_AREA])
if area < component_threshold:
continue
if area > largest_area:
largest_area = area
return float(largest_area / total_area)
def motion_jitter_ratio(shifts: list[tuple[float, float]], diagonal: float) -> float:
if not shifts:
return 0.0
_, np = require_cv2()
normalized_diagonal = max(diagonal, 1.0)
dx_values = [item[0] for item in shifts]
dy_values = [item[1] for item in shifts]
median_dx = float(np.median(dx_values))
median_dy = float(np.median(dy_values))
jitters = [
math.sqrt((dx - median_dx) ** 2 + (dy - median_dy) ** 2) / normalized_diagonal
for dx, dy in shifts
]
return sum(jitters) / max(len(jitters), 1)
def bos_key_join(prefix: str, relative_path: str) -> str:
normalized_prefix = bos_prefix(prefix)
normalized_relative = str(relative_path or "").strip().lstrip("/")
if not normalized_prefix:
return normalized_relative
if not normalized_relative:
return normalized_prefix
return f"{normalized_prefix}/{normalized_relative}"
def require_bos_sdk():
try:
import baidubce.protocol as protocol # type: ignore
from baidubce.auth.bce_credentials import BceCredentials # type: ignore
from baidubce.bce_client_configuration import BceClientConfiguration # type: ignore
from baidubce.services.bos.bos_client import BosClient # type: ignore
except Exception as error: # pragma: no cover - runtime environment specific
raise RuntimeError("BOS 扫描需要 bce-python-sdk 运行时") from error
return protocol, BceCredentials, BceClientConfiguration, BosClient
def build_bos_client(endpoint: str, access_key_id: str, secret_access_key: str):
protocol_module, BceCredentials, BceClientConfiguration, BosClient = require_bos_sdk()
parsed = urlparse(endpoint if "://" in endpoint else f"https://{endpoint}")
scheme = (parsed.scheme or "https").lower()
host = parsed.netloc or parsed.path
if not host:
raise RuntimeError("BOS endpoint 不能为空")
config = BceClientConfiguration(
credentials=BceCredentials(access_key_id, secret_access_key),
endpoint=host,
protocol=protocol_module.HTTPS if scheme == "https" else protocol_module.HTTP,
)
return BosClient(config), scheme
def format_size(num_bytes: int) -> str:
if num_bytes < 1024:
return f"{num_bytes} B"
if num_bytes < 1024 * 1024:
return f"{num_bytes / 1024:.1f} KiB"
if num_bytes < 1024 * 1024 * 1024:
return f"{num_bytes / (1024 * 1024):.1f} MiB"
return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB"
MAX_SHA256_FILE_BYTES = 64 * 1024 * 1024
def sha256_file(path: pathlib.Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(65536), b""):
digest.update(chunk)
return digest.hexdigest()
def artifact_sha256(path: pathlib.Path, size_bytes: int) -> tuple[str, dict[str, Any]]:
if size_bytes > MAX_SHA256_FILE_BYTES:
return "", {"sha256Skipped": True, "sha256SkipReason": "size-limit", "sha256SkipBytes": size_bytes}
return sha256_file(path), {}
def require_cv2() -> tuple[Any, Any]:
try:
import cv2 # type: ignore
import numpy as np # type: ignore
except Exception as error: # pragma: no cover - environment specific
raise RuntimeError("视频算子需要 opencv-python-headless 和 numpy 运行时") from error
return cv2, np
def data_url_for_path(path: pathlib.Path, mime_type: str) -> str:
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f"data:{mime_type};base64,{encoded}"
def decode_data_url(url: str) -> tuple[str, bytes] | None:
if not url.startswith("data:") or "," not in url:
return None
header, encoded = url.split(",", 1)
if ";base64" not in header:
return None
mime_type = header[5:].split(";", 1)[0] or "application/octet-stream"
try:
return mime_type, base64.b64decode(encoded)
except Exception:
return None
def download_remote_source_to_path(url: str, path: pathlib.Path) -> pathlib.Path | None:
parsed = urlparse(url)
if parsed.scheme.lower() not in {"http", "https"}:
return None
path.parent.mkdir(parents=True, exist_ok=True)
remote_request = request.Request(
url,
headers={
"User-Agent": "DroboticFlow operator-runner/1.0",
},
)
try:
with request.urlopen(remote_request, timeout=300) as response, path.open("wb") as handle:
shutil.copyfileobj(response, handle, length=1024 * 1024)
except error.HTTPError as exc:
raise RuntimeError(f"下载远程输入失败({exc.code}): {path.name}") from exc
except error.URLError as exc:
raise RuntimeError(f"下载远程输入失败: {path.name}") from exc
return path
def download_remote_source(url: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
path = output_dir / "_materialized_sources" / default_name
return download_remote_source_to_path(url, path)
def workspace_cached_bos_source_path(raw_path: str, output_dir: pathlib.Path) -> pathlib.Path | None:
bucket, key = parse_bos_path(raw_path)
if not bucket or not key:
return None
workspace_root = str(os.getenv("WORKSPACE_ROOT", "")).strip()
if workspace_root:
root = pathlib.Path(workspace_root)
if root.is_absolute():
return root / "_bos_cache" / bucket / key
return output_dir / "_materialized_sources" / bucket / key
def download_bos_source(raw_path: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
target = workspace_cached_bos_source_path(raw_path, output_dir)
if target is None:
return None
if target.exists() and target.stat().st_size > 0:
return target
bucket, key = parse_bos_path(raw_path)
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
if not bucket or not key or not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
return None
target.parent.mkdir(parents=True, exist_ok=True)
temp_target = target.with_name(target.name + ".part")
with contextlib.suppress(FileNotFoundError):
temp_target.unlink()
try:
client, _ = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
client.get_object_to_file(bucket, key, str(temp_target))
temp_target.replace(target)
except Exception as exc:
with contextlib.suppress(FileNotFoundError):
temp_target.unlink()
raise RuntimeError(f"下载 BOS 输入失败: {default_name}") from exc
return target
def signed_bos_source_url(raw_path: str) -> str:
bucket, key = parse_bos_path(raw_path)
if not bucket or not key:
return ""
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
if not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
return ""
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
return text_value(
client.generate_pre_signed_url(
bucket,
key,
expiration_in_seconds=3600,
protocol=scheme,
)
).strip()
def remote_video_url_for_source(source: dict[str, Any]) -> str:
preview_url = str(source.get("previewURL", "")).strip()
parsed_preview = urlparse(preview_url)
if parsed_preview.scheme.lower() in {"http", "https"}:
return preview_url
raw_path = str(source.get("path", "")).strip()
if raw_path.startswith("bos://"):
return signed_bos_source_url(raw_path)
return ""
def is_archive_path(path: pathlib.Path) -> bool:
name = path.name.lower()
return any(name.endswith(suffix) for suffix in ARCHIVE_FILE_SUFFIXES)
def safe_extract_archive(path: pathlib.Path, destination: pathlib.Path) -> list[str]:
destination.mkdir(parents=True, exist_ok=True)
destination_resolved = destination.resolve()
extracted_files: list[str] = []
def ensure_child(member_name: str) -> None:
target = (destination / member_name).resolve()
if target != destination_resolved and destination_resolved not in target.parents:
raise RuntimeError(f"archive entry escapes target directory: {member_name}")
if path.name.lower().endswith(".zip"):
with zipfile.ZipFile(path) as archive:
for member in archive.infolist():
if not member.filename or member.is_dir():
continue
ensure_child(member.filename)
extracted_files.append(member.filename)
archive.extractall(destination)
return extracted_files
with tarfile.open(path, "r:*") as archive:
members = archive.getmembers()
for member in members:
if not member.name or member.isdir():
continue
ensure_child(member.name)
extracted_files.append(member.name)
archive.extractall(destination)
return extracted_files
def artifact_slug(value: str) -> str:
normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-")
return normalized or "artifact"
def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str:
suffix = path.suffix.lower()
if suffix == ".json":
return "json"
if suffix in VIDEO_FILE_SUFFIXES:
return "video"
if suffix in IMAGE_FILE_SUFFIXES:
return "image"
return fallback
def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None:
cleaned = {key: json_compatible(value) for key, value in metadata.items() if value not in ("", None, [], {})}
return cleaned or None
def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None:
if path.suffix.lower() != ".json":
return None
for attempt in range(PAYLOAD_FILE_READ_ATTEMPTS):
if not path.exists():
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
continue
return None
try:
with path.open("r", encoding="utf-8") as handle:
value = json.load(handle)
except (FileNotFoundError, OSError, json.JSONDecodeError):
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
continue
return None
except Exception:
return None
if isinstance(value, dict):
return value
if isinstance(value, list):
return {"items": value, "itemCount": len(value)}
return {"value": value}
return None
def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None:
outputs = node_data.get("outputs", []) or []
if name:
for output in outputs:
if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name:
return output
if 0 <= index < len(outputs):
return outputs[index]
return None
def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]:
with path.open("rb") as handle:
raw = handle.read(PREVIEW_TEXT_LIMIT + 1)
truncated = len(raw) > PREVIEW_TEXT_LIMIT
text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace")
lines = text.splitlines()
if len(lines) > PREVIEW_TEXT_LINE_LIMIT:
text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT])
truncated = True
line_count = max(1, text.count("\n") + 1) if text else 0
return text, truncated, line_count
def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]:
text, truncated, line_count = read_text_sample(path)
return {
"preview": {
"kind": "text",
"format": format_name,
"text": text,
"truncated": truncated,
"lineCount": line_count,
}
}
def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]:
columns: list[str] = []
rows: list[dict[str, Any]] = []
total_rows = 0
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
reader = csv.reader(handle, delimiter=delimiter)
for index, row in enumerate(reader):
if index == 0:
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
continue
total_rows += 1
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
continue
width = max(len(columns), len(row))
row_payload: dict[str, Any] = {}
for column_index in range(width):
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
rows.append(row_payload)
return {
"preview": {
"kind": "table",
"format": format_name,
"columns": columns,
"rows": rows,
"rowCount": total_rows,
"truncated": total_rows > len(rows),
}
}
def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]:
columns: list[str] = []
rows: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
reader = csv.reader(handle, delimiter=delimiter)
for index, row in enumerate(reader):
if index == 0:
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
continue
width = max(len(columns), len(row))
row_payload: dict[str, Any] = {}
for column_index in range(width):
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
rows.append(row_payload)
return columns, rows
def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]:
rows: list[Any] = []
line_count = 0
with path.open("r", encoding="utf-8", errors="replace") as handle:
for raw_line in handle:
line = raw_line.strip()
if not line:
continue
line_count += 1
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
continue
try:
rows.append(json.loads(line))
except Exception:
rows.append({"value": line})
columns: list[str] = []
for row in rows:
if isinstance(row, dict):
for key in row.keys():
if key not in columns:
columns.append(key)
return {
"preview": {
"kind": "table",
"format": "jsonl",
"columns": columns,
"rows": rows,
"rowCount": line_count,
"truncated": line_count > len(rows),
}
}
def inspect_video_file(path: pathlib.Path) -> dict[str, Any]:
cv2, _ = require_cv2()
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
return {}
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
payload = clean_metadata(
{
"frameRate": round(fps, 3) if fps > 0 else None,
"frameCount": frame_count or None,
"resolution": f"{width}x{height}" if width > 0 and height > 0 else None,
"duration": round(duration, 3) if duration > 0 else None,
}
)
return payload or {}
finally:
capture.release()
def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]:
payload: dict[str, Any] = {
"preview": {
"kind": "image",
"format": path.suffix.lower().lstrip(".") or "image",
}
}
size_bytes = path.stat().st_size if path.exists() else 0
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
payload["preview"]["url"] = data_url_for_path(path, mime_type)
return payload
def preview_media_payload(path: pathlib.Path, mime_type: str, kind: str) -> dict[str, Any]:
preview: dict[str, Any] = {
"kind": kind,
"format": path.suffix.lower().lstrip(".") or kind,
}
if kind == "video":
preview.update(inspect_video_file(path))
size_bytes = path.stat().st_size if path.exists() else 0
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
preview["url"] = data_url_for_path(path, mime_type)
return {"preview": preview}
def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]:
try:
relative = candidate.relative_to(root).as_posix()
except ValueError:
relative = candidate.name
items.append({"path": relative, "sizeBytes": candidate.stat().st_size})
return items
def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]:
files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()]
extensions: dict[str, int] = {}
total_bytes = 0
for candidate in files:
extension = candidate.suffix.lower() or "<none>"
extensions[extension] = extensions.get(extension, 0) + 1
total_bytes += candidate.stat().st_size
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
return {
"root": str(root),
"fileCount": len(files),
"totalBytes": total_bytes,
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
"files": sample_file_entries(files, root),
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
}
def optional_json_file(path: pathlib.Path) -> Any:
if not path.exists():
return None
try:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
except Exception:
return None
def optional_jsonl_rows(path: pathlib.Path) -> list[Any]:
rows: list[Any] = []
if not path.exists():
return rows
try:
with path.open("r", encoding="utf-8", errors="replace") as handle:
for raw_line in handle:
line = raw_line.strip()
if not line:
continue
rows.append(json.loads(line))
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
break
except Exception:
return []
return rows
def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None:
candidates = [path] if path.is_dir() else [path.parent]
candidates.extend(list((path if path.is_dir() else path.parent).parents))
for candidate in candidates:
meta_dir = candidate / "meta"
if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")):
return candidate
return None
def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]:
tree = summarize_directory_tree(root)
meta_dir = root / "meta"
info = optional_json_file(meta_dir / "info.json")
stats = optional_json_file(meta_dir / "stats.json")
tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl")
episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl")
parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()]
video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
return {
"preview": {
"kind": "dataset",
"format": "lerobot",
"root": str(root),
"summary": tree,
"info": info,
"stats": stats,
"tasks": tasks,
"episodes": episodes,
"parquetFiles": sample_file_entries(parquet_files, root),
"parquetCount": len(parquet_files),
"videoCount": len(video_files),
}
}
def sqlite_table_names(connection: sqlite3.Connection) -> list[str]:
rows = connection.execute(
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
).fetchall()
return [str(row[0]) for row in rows]
def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]:
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro&immutable=1", uri=True)
try:
tables = sqlite_table_names(connection)
table_summaries: list[dict[str, Any]] = []
for table_name in tables[:12]:
count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0]
table_summaries.append({"name": table_name, "rowCount": int(count)})
payload: dict[str, Any] = {
"preview": {
"kind": "dataset",
"format": "sqlite",
"tables": table_summaries,
"tableCount": len(tables),
}
}
if "topics" in tables and "messages" in tables:
topic_rows = connection.execute(
"select id, name, type, serialization_format from topics order by id"
).fetchall()
message_counts = {
int(topic_id): int(count)
for topic_id, count in connection.execute(
"select topic_id, count(*) from messages group by topic_id order by topic_id"
).fetchall()
}
payload["preview"]["format"] = "rosbag2"
payload["preview"]["topics"] = [
{
"id": int(topic_id),
"name": str(name),
"type": str(topic_type),
"serialization": str(serialization),
"messageCount": message_counts.get(int(topic_id), 0),
}
for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT]
]
payload["preview"]["messageCount"] = int(sum(message_counts.values()))
return payload
finally:
connection.close()
def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]:
suffix = path.suffix.lower()
if suffix == ".db3":
return summarize_sqlite_payload(path)
header = ""
with path.open("rb") as handle:
header = handle.read(32).decode("utf-8", errors="replace")
format_name = "mcap" if suffix == ".mcap" else "rosbag1"
return {
"preview": {
"kind": "dataset",
"format": format_name,
"header": header.strip(),
"note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。",
}
}
def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None:
direct = params.get("SOURCE_PATH", "").strip()
if direct and pathlib.Path(direct).exists():
return pathlib.Path(direct)
for artifact in inputs:
payload = artifact.get("data")
if isinstance(payload, dict):
candidate = payload.get("sourcePath")
if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists():
return pathlib.Path(candidate)
source_path = artifact.get("path")
if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists():
return pathlib.Path(source_path)
return None
def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None:
data = artifact.get("data")
if not isinstance(data, dict):
return None
preview = data.get("preview")
if isinstance(preview, dict):
return preview
return None
def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]:
extensions: dict[str, int] = {}
files: list[dict[str, Any]] = []
for artifact in inputs:
name = str(artifact.get("name", "") or artifact.get("path", "")).strip()
extension = pathlib.Path(name).suffix.lower() or "<none>"
extensions[extension] = extensions.get(extension, 0) + 1
files.append(
{
"path": str(artifact.get("path", "")).strip() or name,
"name": name,
"size": str(artifact.get("size", "")),
}
)
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
return {
"fileCount": len(files),
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
"files": files[:PREVIEW_FILE_LIST_LIMIT],
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
}
def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str:
for artifact in inputs:
raw_path = str(artifact.get("path", "")).strip()
if not raw_path:
continue
parts = pathlib.PurePosixPath(raw_path).parts
for marker in ("meta", "data", "videos"):
if marker in parts:
index = parts.index(marker)
if index > 0:
return str(pathlib.PurePosixPath(*parts[:index]))
return ""
def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]:
info = None
stats = None
tasks: list[Any] = []
episodes: list[Any] = []
parquet_files: list[dict[str, Any]] = []
video_count = 0
for artifact in inputs:
name = str(artifact.get("name", "")).strip()
path = str(artifact.get("path", "")).strip()
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
preview = artifact_preview_payload(artifact) or {}
if name == "info.json" and isinstance(data, dict):
info = data
elif name == "stats.json" and isinstance(data, dict):
stats = data
elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list):
tasks = preview["rows"]
elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list):
episodes = preview["rows"]
if name.endswith(".parquet"):
parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))})
if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video":
video_count += 1
return {
"preview": {
"kind": "dataset",
"format": "lerobot",
"root": infer_lerobot_root_from_inputs(inputs),
"summary": summarize_input_artifacts(inputs),
"info": info,
"stats": stats,
"tasks": tasks,
"episodes": episodes,
"parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT],
"parquetCount": len(parquet_files),
"videoCount": video_count,
}
}
def parse_positive_float_param(params: dict[str, str], key: str, default: float) -> float:
raw = params.get(key, "").strip()
if not raw:
return default
try:
parsed = float(raw)
except Exception as exc:
raise RuntimeError(f"{key} must be numeric") from exc
if not math.isfinite(parsed):
raise RuntimeError(f"{key} must be finite")
if parsed <= 0:
raise RuntimeError(f"{key} must be greater than 0")
return parsed
def parse_unit_interval_param(params: dict[str, str], key: str, default: float) -> float:
raw = params.get(key, "").strip()
if not raw:
return default
try:
parsed = float(raw)
except Exception as exc:
raise RuntimeError(f"{key} must be numeric") from exc
if not math.isfinite(parsed):
raise RuntimeError(f"{key} must be finite")
if parsed < 0 or parsed > 1:
raise RuntimeError(f"{key} must be in the range [0, 1]")
return parsed
def float_value(value: Any, default: float) -> float:
if value is None:
return default
if isinstance(value, str) and not value.strip():
return default
try:
return float(value)
except Exception:
return default
def parse_positive_int_param(params: dict[str, str], key: str, default: int) -> int:
raw = params.get(key, "").strip()
if not raw:
return default
if not EXACT_POSITIVE_INT_RE.fullmatch(raw):
if INTEGERISH_RE.fullmatch(raw):
try:
parsed_integerish = int(raw)
except Exception:
parsed_integerish = 0
if parsed_integerish <= 0:
raise RuntimeError(f"{key} must be greater than 0")
raise RuntimeError(f"{key} must be an exact positive integer")
raise RuntimeError(f"{key} must be an integer")
parsed = int(raw)
if parsed > MAX_SAFE_INTEGER:
raise RuntimeError(f"{key} must be a safe integer")
return parsed
def parse_bool_param(params: dict[str, str], key: str, default: bool = False) -> bool:
raw = params.get(key, "").strip().lower()
if not raw:
return default
if raw in TRUE_ENV_VALUES:
return True
if raw in FALSE_ENV_VALUES:
return False
raise RuntimeError(f"{key} must be a boolean")
def first_payload_with_keys(inputs: list[dict[str, Any]], *keys: str) -> dict[str, Any] | None:
for artifact in inputs:
payload = load_artifact_payload(artifact)
if isinstance(payload, dict) and all(key in payload for key in keys):
return payload
return None
def common_input_root(paths: list[str]) -> str:
filtered = [item for item in paths if item]
if not filtered:
return ""
if len(filtered) == 1:
return str(pathlib.Path(filtered[0]).parent)
try:
return os.path.commonpath(filtered)
except Exception:
return str(pathlib.Path(filtered[0]).parent)
def artifact_preview_url(artifact: dict[str, Any]) -> str:
preview = artifact_preview_payload(artifact) or {}
value = preview.get("url")
return value if isinstance(value, str) else ""
def source_ref_from_artifact(artifact: dict[str, Any], relative_path: str = "") -> dict[str, Any]:
metadata = artifact.get("metadata") if isinstance(artifact.get("metadata"), dict) else {}
return clean_metadata(
{
"name": str(artifact.get("name", "")).strip(),
"path": str(artifact.get("path", "")).strip(),
"relativePath": relative_path,
"previewURL": artifact_preview_url(artifact),
"mimeType": str(metadata.get("mimeType", "")).strip(),
}
) or {}
def materialize_artifact_to_path(artifact: dict[str, Any], destination: pathlib.Path) -> bool:
raw_path = str(artifact.get("path", "")).strip()
destination.parent.mkdir(parents=True, exist_ok=True)
if raw_path and pathlib.Path(raw_path).exists():
shutil.copy2(raw_path, destination)
return True
preview_url = artifact_preview_url(artifact)
decoded = decode_data_url(preview_url) if preview_url else None
if decoded is not None:
_, payload = decoded
destination.write_bytes(payload)
return True
payload = artifact.get("data")
if isinstance(payload, dict) and destination.suffix.lower() == ".json":
destination.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return True
preview = artifact_preview_payload(artifact) or {}
if preview.get("kind") == "text":
destination.write_text(str(preview.get("text", "")), encoding="utf-8")
return True
return False
def upload_relative_path(artifact: dict[str, Any], input_root: str, fallback_name: str) -> str:
explicit_relative_path = str(artifact.get("relativePath", "")).strip()
if explicit_relative_path:
return pathlib.PurePosixPath(explicit_relative_path).as_posix().lstrip("/")
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
payload_relative_path = str(payload.get("relativePath", "")).strip()
if payload_relative_path:
return pathlib.PurePosixPath(payload_relative_path).as_posix().lstrip("/")
raw_path = str(artifact.get("path", "")).strip()
relative_path = str(fallback_name or artifact.get("name", "")).strip() or "artifact"
if raw_path and input_root:
try:
relative_path = pathlib.Path(raw_path).relative_to(input_root).as_posix()
except Exception:
relative_path = pathlib.Path(raw_path).name or relative_path
if not relative_path:
relative_path = pathlib.Path(fallback_name or "artifact").name or "artifact"
return pathlib.PurePosixPath(relative_path).as_posix().lstrip("/")
def materialized_upload_source(artifact: dict[str, Any], staging_dir: pathlib.Path, relative_path: str) -> pathlib.Path | None:
raw_path = str(artifact.get("path", "")).strip()
if raw_path and os.path.exists(raw_path):
return pathlib.Path(raw_path)
destination = staging_dir / "_materialized_upload_inputs" / relative_path
if destination.exists():
return destination
if materialize_artifact_to_path(artifact, destination):
return destination
return None
def package_config_payload(item: dict[str, Any], manifest_payload: dict[str, Any]) -> dict[str, Any]:
return {
"collectionId": item.get("collectionId", ""),
"meta": item.get("meta", {}),
"segmentDurationSeconds": manifest_payload.get("segmentDurationSeconds"),
"sourceVideo": item.get("sourceVideo", ""),
"startSecond": item.get("startSecond"),
"endSecond": item.get("endSecond"),
"duration": item.get("duration") or item.get("clipDuration"),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
}
def package_related_artifact_names(manifest_payload: dict[str, Any]) -> set[str]:
names: set[str] = set()
for item in manifest_payload.get("items", []):
if not isinstance(item, dict):
continue
clip_id = str(item.get("clipId", "")).strip()
clip_name = str(item.get("clipName", "")).strip()
annotation_name = str(item.get("annotationFile", "")).strip() or (f"{clip_id}_annotation.json" if clip_id else "")
quality_name = f"{clip_id}_quality.json" if clip_id else ""
config_name = f"{clip_id}_config.json" if clip_id else ""
for candidate in (clip_name, annotation_name, quality_name, config_name):
if candidate:
names.add(candidate)
return names
def materialize_package_artifacts_from_manifest(
run_id: str,
node_id: str,
staging_dir: pathlib.Path,
manifest_payload: dict[str, Any],
node_data: dict[str, Any],
) -> list[dict[str, Any]]:
archive_sources = manifest_payload.get("archiveSources", {}) if isinstance(manifest_payload.get("archiveSources"), dict) else {}
binding = output_binding(node_data)
materialized_artifacts: list[dict[str, Any]] = []
materialized_root = staging_dir / "_materialized_upload_inputs"
for item in manifest_payload.get("items", []):
if not isinstance(item, dict):
continue
clip_id = str(item.get("clipId", "")).strip()
if not clip_id:
continue
package_name = str(item.get("packageName", "")).strip() or f"{clip_id}_package"
clip_name = str(item.get("clipName", "")).strip() or f"{clip_id}.mp4"
package_relative_dir = pathlib.PurePosixPath("packages") / package_name
package_dir = materialized_root / package_relative_dir
package_dir.mkdir(parents=True, exist_ok=True)
clip_destination = package_dir / clip_name
direct_clip_path = str(item.get("clipPath", "")).strip()
clip_source: pathlib.Path | None = pathlib.Path(direct_clip_path) if direct_clip_path and pathlib.Path(direct_clip_path).exists() else None
if clip_source is None:
clip_source = materialize_clip_from_source(item, staging_dir, archive_sources)
if clip_source is not None and clip_source.exists():
clip_size = clip_source.stat().st_size
clip_mime_type = mimetypes.guess_type(clip_name)[0] or "video/mp4"
clip_artifact = {
"id": f"{run_id}-{node_id}-{package_name}-{clip_name}",
"runId": run_id,
"nodeId": node_id,
"name": clip_name,
"type": "video",
"size": format_size(clip_size),
"path": str(clip_source),
"metadata": {"mimeType": clip_mime_type},
"data": {
"relativePath": (package_relative_dir / clip_name).as_posix(),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
},
}
clip_artifact["relativePath"] = (package_relative_dir / clip_name).as_posix()
materialized_artifacts.append(clip_artifact)
annotation_name = str(item.get("annotationFile", "")).strip() or f"{clip_id}_annotation.json"
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
annotation_artifact = write_json_artifact(run_id, node_id, package_dir, annotation_name, annotation_payload, output=binding)
annotation_artifact["relativePath"] = (package_relative_dir / annotation_name).as_posix()
materialized_artifacts.append(annotation_artifact)
quality_name = f"{clip_id}_quality.json"
quality_payload = {
"clip_id": clip_id,
"source_video": item.get("sourceVideo", ""),
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
}
quality_artifact = write_json_artifact(run_id, node_id, package_dir, quality_name, quality_payload, output=binding)
quality_artifact["relativePath"] = (package_relative_dir / quality_name).as_posix()
materialized_artifacts.append(quality_artifact)
config_name = f"{clip_id}_config.json"
config_artifact = write_json_artifact(
run_id,
node_id,
package_dir,
config_name,
package_config_payload(item, manifest_payload),
output=binding,
)
config_artifact["relativePath"] = (package_relative_dir / config_name).as_posix()
materialized_artifacts.append(config_artifact)
return materialized_artifacts
def materialize_input_tree(
inputs: list[dict[str, Any]],
output_dir: pathlib.Path,
) -> tuple[pathlib.Path | None, dict[str, dict[str, Any]]]:
paths = [str(item.get("path", "")).strip() for item in inputs if str(item.get("path", "")).strip()]
root = common_input_root(paths)
if not root:
return None, {}
target_root = output_dir / "_materialized_inputs"
if target_root.exists():
shutil.rmtree(target_root)
target_root.mkdir(parents=True, exist_ok=True)
source_refs: dict[str, dict[str, Any]] = {}
for index, artifact in enumerate(inputs):
raw_path = str(artifact.get("path", "")).strip()
relative_path = str(artifact.get("name", "")).strip() or f"artifact-{index}"
if raw_path:
try:
relative_path = pathlib.Path(raw_path).relative_to(root).as_posix()
except Exception:
relative_path = pathlib.Path(raw_path).name or relative_path
destination = target_root / relative_path
if materialize_artifact_to_path(artifact, destination):
source_refs[relative_path] = source_ref_from_artifact(artifact, relative_path)
if destination.is_file() and is_archive_path(destination):
extracted_files = safe_extract_archive(destination, target_root)
archive_preview_url = artifact_preview_url(artifact)
archive_key = artifact_slug(hashlib.sha256(archive_preview_url.encode("utf-8")).hexdigest()[:16]) if archive_preview_url else artifact_slug(relative_path)
for extracted_relative_path in extracted_files:
normalized_relative_path = pathlib.PurePosixPath(extracted_relative_path).as_posix().lstrip("/")
source_refs[normalized_relative_path] = clean_metadata(
{
"name": pathlib.PurePosixPath(normalized_relative_path).name,
"path": str(target_root / normalized_relative_path),
"relativePath": normalized_relative_path,
"archiveKey": archive_key,
"archivePreviewURL": archive_preview_url,
"archiveMemberPath": normalized_relative_path,
"archiveName": destination.name,
"mimeType": mimetypes.guess_type(normalized_relative_path)[0] or "",
}
) or {}
return target_root, source_refs
def detect_xplatform_collections(root: pathlib.Path) -> list[pathlib.Path]:
if not root.exists():
return []
candidates: list[pathlib.Path] = []
if (root / "meta.json").exists():
return [root]
for child in sorted(root.iterdir()):
if not child.is_dir():
continue
if (child / "meta.json").exists():
candidates.append(child)
continue
if any(candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES for candidate in child.rglob("*")):
candidates.append(child)
return candidates
def normalize_collection_relative_path(value: pathlib.PurePath | str) -> str:
text = value.as_posix() if isinstance(value, pathlib.PurePath) else str(value or "")
normalized = text.strip().strip("/")
if normalized in {"", "."}:
return ""
return normalized
def collection_name_from_source_path(source_path: str, fallback_name: str = "collection") -> str:
candidate = pathlib.PurePosixPath(str(source_path or "").rstrip("/")).name
return candidate or fallback_name
def parse_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
rows: dict[str, dict[str, Any]] = {}
if not isinstance(value, list):
return rows
for item in value:
if not isinstance(item, dict):
continue
filename = str(item.get("filename", "")).strip()
if not filename:
continue
rows[filename] = item
return rows
def collection_file_entry(
root: pathlib.Path,
collection_dir: pathlib.Path,
candidate: pathlib.Path,
collection_id: str,
meta: dict[str, Any],
video_meta_rows: dict[str, dict[str, Any]],
) -> dict[str, Any]:
relative_path = candidate.relative_to(root).as_posix()
collection_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
collection_relative_file_path = candidate.relative_to(collection_dir).as_posix()
mime_type = mimetypes.guess_type(candidate.name)[0] or ""
entry: dict[str, Any] = {
"collectionId": collection_id,
"collectionName": collection_dir.name,
"collectionPath": str(collection_dir),
"collectionRelativePath": collection_relative_path,
"meta": meta,
"relativePath": relative_path,
"path": str(candidate),
"sizeBytes": candidate.stat().st_size,
"mimeType": mime_type,
"source": {"path": str(candidate), "relativePath": relative_path, "mimeType": mime_type},
}
if candidate.suffix.lower() == ".json":
payload = optional_json_file(candidate)
if payload is not None:
entry["data"] = payload
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
matched_meta = video_meta_rows.get(candidate.name, {})
entry.update(
{
"videoFolder": collection_relative_file_path.split("/", 1)[0] if "/" in collection_relative_file_path else "",
"filename": candidate.name,
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
}
)
return entry
def collection_summary(
root: pathlib.Path,
collection_dir: pathlib.Path,
meta: dict[str, Any],
video_meta_rows: dict[str, dict[str, Any]],
) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
files = [candidate for candidate in sorted(collection_dir.rglob("*")) if candidate.is_file()]
videos = [candidate for candidate in files if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
total_bytes = sum(candidate.stat().st_size for candidate in files)
required = ["meta.json", "video_meta.json"]
missing_required = []
if not (collection_dir / "meta.json").exists():
missing_required.append("meta.json")
if not any((collection_dir / candidate).exists() for candidate in XPLATFORM_VIDEO_META_FILES):
missing_required.append("video_meta.json")
has_intrinsics = (collection_dir / "intrinsics.json").exists()
collection_root_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
collection_id = collection_key(collection_root_relative_path, collection_dir.name)
items: list[dict[str, Any]] = []
entries: list[dict[str, Any]] = []
for candidate in files:
entry = collection_file_entry(root, collection_dir, candidate, collection_id, meta, video_meta_rows)
entries.append(entry)
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
items.append(dict(entry))
summary = {
"id": collection_id,
"name": collection_dir.name,
"path": str(collection_dir),
"relativePath": collection_root_relative_path,
"meta": meta,
"hasIntrinsics": has_intrinsics,
"hasVideoMeta": bool(video_meta_rows),
"requiredFiles": required,
"missingRequiredFiles": missing_required,
"videoCount": len(videos),
"fileCount": len(files),
"totalBytes": total_bytes,
"valid": len(missing_required) == 0 and len(videos) > 0,
}
return summary, items, entries
def source_path_join(root: str, relative_path: str) -> str:
if not root:
return relative_path
return root.rstrip("/") + "/" + relative_path.lstrip("/")
def manifest_source_ref(entry: dict[str, Any], relative_path: str, source_path: str) -> dict[str, Any]:
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
storage_path = str(source.get("path", "") or entry.get("storagePath", "") or source_path_join(source_path, relative_path)).strip()
return clean_metadata(
{
"path": storage_path,
"relativePath": relative_path,
"previewURL": str(source.get("previewURL", "")).strip(),
"mimeType": str(entry.get("mimeType", "")).strip(),
}
) or {}
def manifest_entry_json_payload(entry: Any, source_path: str = "") -> Any:
if not isinstance(entry, dict):
return None
inline_data = entry.get("data")
if inline_data is not None:
return inline_data
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
candidates = [str(source.get("path", "")).strip(), str(entry.get("storagePath", "")).strip(), str(entry.get("path", "")).strip()]
for raw_candidate in candidates:
if not raw_candidate:
continue
if "://" in raw_candidate and not raw_candidate.startswith("file://"):
continue
candidate_text = raw_candidate[len("file://") :] if raw_candidate.startswith("file://") else raw_candidate
candidate_path = pathlib.Path(candidate_text)
if not candidate_path.is_absolute():
normalized_source_path = str(source_path or "").strip()
if normalized_source_path and "://" not in normalized_source_path:
candidate_path = pathlib.Path(normalized_source_path) / candidate_text
if candidate_path.is_file():
return optional_json_file(candidate_path)
return None
def manifest_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
if isinstance(value, dict) and isinstance(value.get("items"), list):
return parse_video_meta_rows(value.get("items"))
return parse_video_meta_rows(value)
def collection_key(relative_path: str, fallback_name: str = "collection") -> str:
normalized = relative_path.strip().strip("/")
if not normalized:
return artifact_slug(fallback_name)
return artifact_slug(normalized)
def fallback_manifest_collections(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
grouped: dict[str, list[dict[str, Any]]] = {}
for entry in entries:
relative_path = str(entry.get("path", "")).strip().lstrip("/")
if not relative_path:
continue
parts = pathlib.PurePosixPath(relative_path).parts
if len(parts) < 2:
continue
grouped.setdefault(parts[0], []).append({**entry, "path": relative_path})
collection_summaries: list[dict[str, Any]] = []
items: list[dict[str, Any]] = []
indexed_entries: list[dict[str, Any]] = []
for collection_relative_path in sorted(grouped.keys()):
collection_entries = grouped[collection_relative_path]
collection_name = pathlib.PurePosixPath(collection_relative_path).name
collection_id = collection_key(collection_relative_path, collection_name)
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/meta.json"), None)
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/intrinsics.json"), None)
video_meta_entry = next(
(
entry
for entry in collection_entries
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
),
None,
)
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
meta = meta_payload if isinstance(meta_payload, dict) else {}
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
videos = [
entry
for entry in collection_entries
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
]
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
required = ["meta.json", "video_meta.json"]
missing_required = []
if meta_entry is None:
missing_required.append("meta.json")
if video_meta_entry is None:
missing_required.append("video_meta.json")
collection_path = source_path_join(source_path, collection_relative_path)
collection_summaries.append(
{
"id": collection_id,
"name": collection_name,
"path": collection_path,
"relativePath": collection_relative_path,
"meta": meta,
"hasIntrinsics": intrinsics_entry is not None,
"hasVideoMeta": bool(video_meta_rows),
"requiredFiles": required,
"missingRequiredFiles": missing_required,
"videoCount": len(videos),
"fileCount": len(collection_entries),
"totalBytes": total_bytes,
"valid": len(missing_required) == 0 and len(videos) > 0,
}
)
for collection_entry in collection_entries:
relative_path = str(collection_entry.get("path", "")).strip()
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
indexed_entry = {
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
"mimeType": entry_mime_type,
"source": manifest_source_ref(collection_entry, relative_path, source_path),
}
if "data" in collection_entry:
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
indexed_entry.update(
{
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
}
)
indexed_entries.append(indexed_entry)
for video_entry in videos:
relative_path = str(video_entry.get("path", "")).strip()
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
items.append(
{
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
"source": manifest_source_ref(video_entry, relative_path, source_path),
}
)
return {
"datasetType": "x-platform",
"rootPath": source_path,
"collectionCount": len(collection_summaries),
"videoCount": len(items),
"collections": collection_summaries,
"items": items,
"entries": indexed_entries,
}
def scan_xplatform_manifest_entries(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
normalized_entries: list[dict[str, Any]] = []
meta_roots: set[str] = set()
auxiliary_roots: set[str] = set()
has_root_metadata = False
for entry in entries:
relative_path = str(entry.get("path", "")).strip().lstrip("/")
if not relative_path:
continue
normalized_entry = {**entry, "path": relative_path}
normalized_entries.append(normalized_entry)
path_obj = pathlib.PurePosixPath(relative_path)
parent = path_obj.parent.as_posix()
if parent in {".", ""}:
parent = ""
file_name = path_obj.name.lower()
if file_name == "meta.json":
if parent:
meta_roots.add(parent)
else:
has_root_metadata = True
elif is_xplatform_video_meta_name(file_name) or file_name == "intrinsics.json":
if parent:
auxiliary_roots.add(parent)
else:
has_root_metadata = True
root_candidates = meta_roots or auxiliary_roots
if has_root_metadata:
root_candidates = set(root_candidates)
root_candidates.add("")
if not root_candidates:
return fallback_manifest_collections(normalized_entries, source_path)
ordered_roots = sorted(root_candidates, key=lambda value: (-len(pathlib.PurePosixPath(value).parts), value))
grouped: dict[str, list[dict[str, Any]]] = {root: [] for root in ordered_roots}
unmatched_entries: list[dict[str, Any]] = []
for entry in normalized_entries:
relative_path = str(entry.get("path", "")).strip()
matched_root = next(
(
root
for root in ordered_roots
if (
(not root)
or relative_path == root
or relative_path.startswith(root + "/")
)
),
None,
)
if matched_root is not None:
grouped.setdefault(matched_root, []).append(entry)
else:
unmatched_entries.append(entry)
collection_summaries: list[dict[str, Any]] = []
items: list[dict[str, Any]] = []
indexed_entries: list[dict[str, Any]] = []
for collection_relative_path in sorted(grouped.keys()):
collection_entries = grouped[collection_relative_path]
if not collection_entries:
continue
collection_name = (
pathlib.PurePosixPath(collection_relative_path).name
if collection_relative_path
else collection_name_from_source_path(source_path)
)
collection_id = collection_key(collection_relative_path, collection_name)
meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json"
intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json"
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None)
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None)
video_meta_entry = next(
(
entry
for entry in collection_entries
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
),
None,
)
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
meta = meta_payload if isinstance(meta_payload, dict) else {}
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
videos = [
entry
for entry in collection_entries
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
]
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
required = ["meta.json", "video_meta.json"]
missing_required = []
if meta_entry is None:
missing_required.append("meta.json")
if video_meta_entry is None:
missing_required.append("video_meta.json")
collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path)
collection_summaries.append(
{
"id": collection_id,
"name": collection_name,
"path": collection_path,
"relativePath": collection_relative_path,
"meta": meta,
"hasIntrinsics": intrinsics_entry is not None,
"hasVideoMeta": bool(video_meta_rows),
"requiredFiles": required,
"missingRequiredFiles": missing_required,
"videoCount": len(videos),
"fileCount": len(collection_entries),
"totalBytes": total_bytes,
"valid": len(missing_required) == 0 and len(videos) > 0,
}
)
for collection_entry in collection_entries:
relative_path = str(collection_entry.get("path", "")).strip()
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
indexed_entry = {
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
"mimeType": entry_mime_type,
"source": manifest_source_ref(collection_entry, relative_path, source_path),
}
if "data" in collection_entry:
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
indexed_entry.update(
{
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
}
)
indexed_entries.append(indexed_entry)
for video_entry in videos:
relative_path = str(video_entry.get("path", "")).strip()
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
items.append(
{
"collectionId": collection_id,
"collectionName": collection_name,
"collectionPath": collection_path,
"collectionRelativePath": collection_relative_path,
"meta": meta,
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
"filename": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": source_path_join(source_path, relative_path),
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
"codec": matched_meta.get("codec"),
"resolution": matched_meta.get("resolution"),
"frameRate": matched_meta.get("frameRate"),
"duration": matched_meta.get("duration"),
"startTime": matched_meta.get("startTime"),
"source": manifest_source_ref(video_entry, relative_path, source_path),
}
)
if unmatched_entries:
fallback_payload = fallback_manifest_collections(unmatched_entries, source_path)
collection_summaries.extend(fallback_payload.get("collections", []))
items.extend(fallback_payload.get("items", []))
indexed_entries.extend(fallback_payload.get("entries", []))
return {
"datasetType": "x-platform",
"rootPath": source_path,
"collectionCount": len(collection_summaries),
"videoCount": len(items),
"collections": collection_summaries,
"items": items,
"entries": indexed_entries,
}
def with_xplatform_source_context(payload: dict[str, Any], source_manifest: dict[str, Any]) -> dict[str, Any]:
for key in (
"dataSource",
"sourcePath",
"scanMode",
"listedCount",
"newCount",
"sourceFingerprint",
"bucket",
"prefix",
"pattern",
"endpoint",
):
if key in source_manifest:
payload[key] = source_manifest.get(key)
return payload
def attach_xplatform_catalog_payload(payload: dict[str, Any], catalog_payload: dict[str, Any]) -> dict[str, Any]:
payload["catalogCollectionCount"] = int(catalog_payload.get("collectionCount", 0) or 0)
payload["catalogVideoCount"] = int(catalog_payload.get("videoCount", 0) or 0)
payload["catalogCollections"] = catalog_payload.get("collections", [])
payload["catalogItems"] = catalog_payload.get("items", [])
payload["catalogEntries"] = catalog_payload.get("entries", [])
return payload
def apply_source_refs_to_items(
items: list[dict[str, Any]],
source_refs: dict[str, dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]:
if not source_refs:
return items, {}
archive_sources: dict[str, dict[str, Any]] = {}
for item in items:
relative_path = str(item.get("relativePath", "")).strip()
if not relative_path:
continue
source_ref = source_refs.get(relative_path)
if source_ref:
archive_key = str(source_ref.get("archiveKey", "")).strip()
archive_preview_url = str(source_ref.get("archivePreviewURL", "")).strip()
if archive_key and archive_preview_url and archive_key not in archive_sources:
archive_sources[archive_key] = clean_metadata(
{
"name": str(source_ref.get("archiveName", "")).strip() or "archive",
"previewURL": archive_preview_url,
}
) or {}
item_source = {**source_ref, "path": item.get("path", "")}
item_source.pop("archivePreviewURL", None)
item_source.pop("archiveName", None)
item["source"] = item_source
return items, archive_sources
def scan_xplatform_dataset(root: pathlib.Path, source_refs: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]:
collections = detect_xplatform_collections(root)
collection_summaries: list[dict[str, Any]] = []
items: list[dict[str, Any]] = []
entries: list[dict[str, Any]] = []
for collection_dir in collections:
meta = optional_json_file(collection_dir / "meta.json")
meta = meta if isinstance(meta, dict) else {}
video_meta_rows, _ = load_collection_video_meta(collection_dir)
summary, collection_items, collection_entries = collection_summary(root, collection_dir, meta, video_meta_rows)
collection_summaries.append(summary)
items.extend(collection_items)
entries.extend(collection_entries)
items, archive_sources = apply_source_refs_to_items(items, source_refs or {})
entries, entry_archive_sources = apply_source_refs_to_items(entries, source_refs or {})
archive_sources.update(entry_archive_sources)
payload = {
"datasetType": "x-platform",
"rootPath": str(root),
"collectionCount": len(collection_summaries),
"videoCount": len(items),
"collections": collection_summaries,
"items": items,
"entries": entries,
}
if archive_sources:
payload["archiveSources"] = archive_sources
return payload
def load_xplatform_index_payload(
params: dict[str, str],
inputs: list[dict[str, Any]],
output_dir: pathlib.Path,
) -> dict[str, Any]:
source_manifest = first_payload_with_keys(inputs, "entries")
if source_manifest is not None:
entries = source_manifest.get("entries")
if isinstance(entries, list):
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
payload = scan_xplatform_manifest_entries(entries, manifest_source_path)
catalog_entries = source_manifest.get("catalogEntries")
if isinstance(catalog_entries, list):
catalog_payload = scan_xplatform_manifest_entries(catalog_entries, manifest_source_path)
attach_xplatform_catalog_payload(payload, catalog_payload)
return with_xplatform_source_context(payload, source_manifest)
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
if manifest_source_path and pathlib.Path(manifest_source_path).exists():
root = pathlib.Path(manifest_source_path)
return scan_xplatform_dataset(root if root.is_dir() else root.parent)
for artifact in inputs:
name = str(artifact.get("name", "")).strip().lower()
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
if name != "source-manifest.json" and not any(key in data for key in ("entriesCount", "catalogEntriesCount", "sourcePath")):
continue
raise RuntimeError("X 平台数据集索引未能读取 source-manifest 的 entries,拒绝回退到空目录扫描")
source_path = resolve_source_path(params, inputs)
if source_path is not None:
root = source_path if source_path.is_dir() else source_path.parent
return scan_xplatform_dataset(root)
materialized_root, source_refs = materialize_input_tree(inputs, output_dir)
if materialized_root is not None:
return scan_xplatform_dataset(materialized_root, source_refs)
raise RuntimeError("X 平台数据集索引需要 SOURCE_PATH 或上游输入提供真实数据目录")
def normalize_date_value(value: Any) -> str:
text = str(value or "").strip()
digits = "".join(character for character in text if character.isdigit())
if len(digits) >= 8:
return digits[:8]
return ""
def date_in_range(value: Any, start: str, end: str) -> bool:
normalized = normalize_date_value(value)
if not normalized:
return False
if start and normalized < start:
return False
if end and normalized > end:
return False
return True
def materialize_archive_member(
source: dict[str, Any],
archive_sources: dict[str, Any],
default_name: str,
output_dir: pathlib.Path,
) -> pathlib.Path | None:
archive_key = str(source.get("archiveKey", "")).strip()
archive_source = archive_sources.get(archive_key) if isinstance(archive_sources, dict) else {}
archive_source = archive_source if isinstance(archive_source, dict) else {}
preview_url = str(archive_source.get("previewURL", "")).strip()
member_path = str(source.get("archiveMemberPath", "")).strip()
decoded = decode_data_url(preview_url) if preview_url and member_path else None
if decoded is None:
return None
_, payload = decoded
archive_name = pathlib.PurePosixPath(str(archive_source.get("name", "")).strip() or "archive").name
archive_path = output_dir / "_materialized_sources" / archive_name
archive_path.parent.mkdir(parents=True, exist_ok=True)
archive_path.write_bytes(payload)
destination = output_dir / "_materialized_sources" / default_name
destination.parent.mkdir(parents=True, exist_ok=True)
if archive_name.lower().endswith(".zip"):
with zipfile.ZipFile(archive_path) as archive:
with archive.open(member_path) as handle:
destination.write_bytes(handle.read())
return destination
with tarfile.open(archive_path, "r:*") as archive:
extracted = archive.extractfile(member_path)
if extracted is None:
return None
with extracted:
destination.write_bytes(extracted.read())
return destination
def source_path_from_item(
item: dict[str, Any],
default_name: str,
output_dir: pathlib.Path,
archive_sources: dict[str, Any] | None = None,
) -> pathlib.Path:
source = item.get("source") if isinstance(item.get("source"), dict) else {}
direct = str(source.get("path", "") or item.get("path", "")).strip()
if direct and pathlib.Path(direct).exists():
return pathlib.Path(direct)
if direct.startswith("bos://"):
cached_target = workspace_cached_bos_source_path(direct, output_dir)
if cached_target is not None and cached_target.exists() and cached_target.stat().st_size > 0:
return cached_target
cached_bos_source = download_bos_source(direct, default_name, output_dir)
if cached_bos_source is not None:
return cached_bos_source
preview_url = str(source.get("previewURL", "")).strip()
if cached_target is not None and preview_url:
cached_remote_source = download_remote_source_to_path(preview_url, cached_target)
if cached_remote_source is not None:
return cached_remote_source
archive_member = materialize_archive_member(source, archive_sources or {}, default_name, output_dir)
if archive_member is not None:
return archive_member
preview_url = str(source.get("previewURL", "")).strip()
remote_path = download_remote_source(preview_url, default_name, output_dir) if preview_url else None
if remote_path is not None:
return remote_path
decoded = decode_data_url(preview_url) if preview_url else None
if decoded is None:
raise RuntimeError(f"无法获取真实输入文件: {default_name}")
_, payload = decoded
path = output_dir / "_materialized_sources" / default_name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(payload)
return path
def clip_segment_bounds(item: dict[str, Any]) -> tuple[float, float]:
start_second = float(item.get("startSecond") or 0.0)
end_second = float(item.get("endSecond") or 0.0)
duration = float(item.get("duration") or 0.0)
if end_second <= start_second and duration > 0:
end_second = start_second + duration
if end_second <= start_second:
end_second = start_second + 0.1
return start_second, end_second
def materialize_clip_from_source(
item: dict[str, Any],
output_dir: pathlib.Path,
archive_sources: dict[str, Any] | None = None,
) -> pathlib.Path | None:
source = item.get("source") if isinstance(item.get("source"), dict) else {}
source_name = (
str(item.get("sourceVideo", "")).strip()
or pathlib.PurePosixPath(str(source.get("path", "")).strip()).name
or "source.mp4"
)
clip_name = str(item.get("clipName", "")).strip() or f"{pathlib.Path(source_name).stem}_clip.mp4"
destination = output_dir / "_materialized_clips" / clip_name
if destination.exists():
return destination
start_second, end_second = clip_segment_bounds(item)
remote_source_url = remote_video_url_for_source(source) if source else ""
if remote_source_url:
try:
write_video_segment_from_remote_url(remote_source_url, destination, start_second, end_second)
return destination
except RuntimeError as exc:
log(f"[operator-runner][warn] fallback to temporary source materialization for {clip_name}: {exc}")
if not source:
return None
source_path = source_path_from_item({"source": source}, source_name, output_dir, archive_sources)
write_video_segment(source_path, destination, start_second, end_second)
return destination
def analyze_video_quality_metrics(path: pathlib.Path, clarity_scale: float) -> dict[str, Any]:
cv2, np = require_cv2()
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频文件: {path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
positions = sampled_frame_positions(frame_count)
if not positions:
positions = [0]
clarities: list[float] = []
shifts: list[tuple[float, float]] = []
skin_ratios: list[float] = []
previous_gray = None
previous_position: int | None = None
analysis_diagonal = math.sqrt((width or 1) ** 2 + (height or 1) ** 2)
for position in positions:
if frame_count > 0:
capture.set(cv2.CAP_PROP_POS_FRAMES, position)
ok, frame = capture.read()
if not ok:
continue
analysis_frame = resize_quality_frame(frame)
analysis_height, analysis_width = analysis_frame.shape[:2]
analysis_diagonal = math.sqrt((analysis_width or 1) ** 2 + (analysis_height or 1) ** 2)
gray = cv2.cvtColor(analysis_frame, cv2.COLOR_BGR2GRAY)
laplacian = cv2.Laplacian(gray, cv2.CV_64F).var()
clarities.append(float(laplacian))
if previous_gray is not None:
gap = max(position - previous_position, 1) if previous_position is not None else 1
shift, _ = cv2.phaseCorrelate(np.float32(previous_gray), np.float32(gray))
shifts.append((float(shift[0]) / gap, float(shift[1]) / gap))
mask = build_skin_mask(analysis_frame)
skin_ratios.append(largest_skin_component_ratio(mask))
previous_gray = gray
previous_position = position
clarity_raw = sum(clarities) / max(len(clarities), 1)
clarity = 1.0 - math.exp(-max(clarity_raw, 0.0) / clarity_scale)
shake = motion_jitter_ratio(shifts, analysis_diagonal) if shifts else 0.0
hand_occlusion = sum(skin_ratios) / max(len(skin_ratios), 1) if skin_ratios else 0.0
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
quality_score = max(0.0, min(1.0, (clarity + (1.0 - min(shake, 1.0)) + (1.0 - min(hand_occlusion, 1.0))) / 3.0))
return {
"codec": "opencv",
"resolution": f"{width}x{height}" if width > 0 and height > 0 else "",
"frameRate": round(fps, 3) if fps > 0 else None,
"duration": round(duration, 3) if duration > 0 else None,
"clarity": round(clarity, 4),
"shake": round(shake, 4),
"handOcclusion": round(hand_occlusion, 4),
"qualityScore": round(quality_score, 4),
"sampleCount": len(clarities),
}
finally:
capture.release()
def stable_sample_hit(value: str, percent: float) -> bool:
if percent >= 100:
return True
if percent <= 0:
return False
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
score = int(digest[:8], 16) / 0xFFFFFFFF
return score <= (percent / 100.0)
def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None:
suffix = path.suffix.lower()
if suffix in TABULAR_FILE_SUFFIXES:
return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip("."))
if suffix == ".jsonl":
return preview_jsonl_payload(path)
if suffix in TEXT_FILE_SUFFIXES:
format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".")
return preview_text_payload(path, format_name)
if suffix in IMAGE_FILE_SUFFIXES:
return preview_image_payload(path, mime_type)
if suffix in ROSBAG_FILE_SUFFIXES:
return summarize_rosbag_file(path)
if suffix == ".pdf":
return {"preview": {"kind": "document", "format": "pdf"}}
if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES:
return preview_media_payload(path, mime_type or "video/mp4", "video")
if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
return preview_media_payload(path, mime_type or "audio/mpeg", "audio")
size_bytes = path.stat().st_size if path.exists() else 0
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
return {"preview": {"kind": "binary", "format": suffix.lstrip(".") or "file", "url": data_url_for_path(path, mime_type)}}
return None
def build_artifact(
run_id: str,
node_id: str,
path: pathlib.Path,
artifact_type: str,
data: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
mime_type, _ = mimetypes.guess_type(path.name)
size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8"))
sha256_value = ""
sha256_metadata: dict[str, Any] = {}
if path.exists():
sha256_value, sha256_metadata = artifact_sha256(path, size_bytes)
artifact_metadata = clean_metadata(
{
**(metadata or {}),
**sha256_metadata,
"sha256": sha256_value,
"mimeType": mime_type or "",
}
)
artifact = {
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}",
"runId": run_id,
"nodeId": node_id,
"name": path.name,
"type": artifact_type,
"size": format_size(size_bytes),
"path": str(path),
}
if output:
if str(output.get("id", "")).strip():
artifact["portId"] = str(output.get("id", "")).strip()
if str(output.get("name", "")).strip():
artifact["portName"] = str(output.get("name", "")).strip()
if artifact_metadata:
artifact["metadata"] = artifact_metadata
if data:
artifact["data"] = data
return artifact
def build_virtual_artifact(
run_id: str,
node_id: str,
name: str,
path: str,
artifact_type: str,
size_bytes: int,
data: dict[str, Any] | None = None,
metadata: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
artifact = {
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else name)}-{artifact_slug(name)}",
"runId": run_id,
"nodeId": node_id,
"name": name,
"type": artifact_type,
"size": format_size(max(0, size_bytes)),
"path": path,
}
if output:
if str(output.get("id", "")).strip():
artifact["portId"] = str(output.get("id", "")).strip()
if str(output.get("name", "")).strip():
artifact["portName"] = str(output.get("name", "")).strip()
if metadata:
cleaned = clean_metadata(metadata)
if cleaned:
artifact["metadata"] = cleaned
if data:
artifact["data"] = data
return artifact
def dataset_partition_keys(payload: Any) -> list[str]:
if not isinstance(payload, dict):
return []
keys: list[str] = []
for key in DATASET_PARTITION_KEYS:
if isinstance(payload.get(key), list):
keys.append(key)
return keys
def should_externalize_structured_payload(payload: Any) -> bool:
if not isinstance(payload, dict):
return False
if dataset_partition_keys(payload):
return True
dataset_type = str(payload.get("datasetType", "")).strip()
if dataset_type:
return True
return False
def dataset_payload_summary(payload: Any) -> dict[str, Any]:
payload = normalize_json_payload(payload)
summary: dict[str, Any] = {}
for key in (
"datasetType",
"count",
"itemCount",
"collectionCount",
"entryCount",
"videoCount",
"clipCount",
"annotationCount",
"packageCount",
"sampledCount",
"passedCount",
"failedCount",
"validCollectionCount",
"invalidCollectionCount",
"dataSource",
"sourcePath",
"bucket",
"prefix",
"endpoint",
"pattern",
"scanMode",
"sourceFingerprint",
"listedCount",
"newCount",
"rootPath",
"segmentDurationSeconds",
):
if key in payload:
summary[key] = json_compatible(payload[key])
if "preview" in payload:
summary["preview"] = json_compatible(payload.get("preview"))
for key in dataset_partition_keys(payload):
value = payload.get(key)
if isinstance(value, list):
summary[f"{key}Count"] = len(value)
if "preview" not in summary:
summary["preview"] = {
"kind": "dataset",
"format": str(payload.get("datasetType", "")).strip() or "json",
"counts": {key: summary[key] for key in summary.keys() if key.endswith("Count") or key == "count"},
}
return summary
def chunk_list(values: list[Any], chunk_size: int = DATASET_PARTITION_SIZE) -> list[list[Any]]:
if chunk_size <= 0:
chunk_size = DATASET_PARTITION_SIZE
return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] or [[]]
def externalize_structured_payload(path: pathlib.Path, payload: dict[str, Any]) -> dict[str, Any]:
normalized_payload = json_compatible(payload)
dataset_dir = path.parent / "_datasets" / artifact_slug(path.stem or path.name)
partitions_dir = dataset_dir / "partitions"
partitions_dir.mkdir(parents=True, exist_ok=True)
partition_files: dict[str, list[str]] = {}
partition_count = 0
partition_keys = dataset_partition_keys(normalized_payload)
for key in partition_keys:
values = normalized_payload.get(key)
if not isinstance(values, list):
continue
files: list[str] = []
for index, chunk in enumerate(chunk_list(values), start=1):
partition_path = partitions_dir / f"{key}-p{index:06d}.json"
write_json_file(partition_path, chunk)
files.append(str(partition_path))
partition_files[key] = files
partition_count += len(files)
manifest = {
"kind": "dataset",
"datasetType": str(normalized_payload.get("datasetType", "")).strip() or "json",
"payloadPath": str(path),
"partitionDir": str(partitions_dir),
"partitionKeys": partition_keys,
"partitionFiles": partition_files,
"summary": dataset_payload_summary(normalized_payload),
"generatedAt": now_iso(),
}
manifest_path = write_json_file(dataset_dir / "manifest.json", manifest)
data_ref = {
"kind": "dataset",
"datasetType": manifest["datasetType"],
"manifestPath": str(manifest_path),
"payloadPath": str(path),
"partitionDir": str(partitions_dir),
"partitionCount": partition_count,
}
item_count = 0
for key in partition_keys:
values = normalized_payload.get(key)
if isinstance(values, list):
item_count += len(values)
if item_count > 0:
data_ref["itemCount"] = item_count
summary = dataset_payload_summary(normalized_payload)
summary["dataRef"] = data_ref
return summary
def write_json_artifact(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
filename: str,
payload: Any,
artifact_type: str = "json",
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
normalized_payload = json_compatible(payload)
path = output_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
json.dump(normalized_payload, handle, ensure_ascii=False, indent=2)
artifact_data: dict[str, Any] | None = normalize_json_payload(normalized_payload)
if isinstance(normalized_payload, dict) and should_externalize_structured_payload(normalized_payload):
artifact_data = externalize_structured_payload(path, normalized_payload)
return build_artifact(run_id, node_id, path, artifact_type, data=artifact_data, output=output)
def summarize_json_like(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return {"kind": "object", "keys": sorted(value.keys())}
if isinstance(value, list):
return {"kind": "list", "length": len(value)}
return {"kind": type(value).__name__, "value": value}
def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]:
data = artifact.get("data")
if isinstance(data, dict):
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
payload_path = str(data_ref.get("payloadPath", "")).strip()
if payload_path:
payload = load_payload_from_file(pathlib.Path(payload_path))
if payload is not None:
return normalize_json_payload(payload)
if isinstance(data, dict):
return data
path_value = artifact.get("path")
if isinstance(path_value, str) and path_value:
payload = load_payload_from_file(pathlib.Path(path_value))
if payload is not None:
return normalize_json_payload(payload)
summary = {
"name": artifact.get("name", ""),
"type": artifact.get("type", ""),
"path": artifact.get("path", ""),
"size": artifact.get("size", ""),
}
return {key: value for key, value in summary.items() if value}
def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]:
merged: dict[str, Any] = {}
for value in values:
merged.update(value)
return merged
def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]:
entries: list[dict[str, Any]] = []
if not base_path.exists():
return entries
if base_path.is_file():
relative = base_path.name
if pattern and not fnmatch.fnmatch(relative, pattern):
return entries
return [
{
"path": relative,
"sizeBytes": base_path.stat().st_size,
"sha256": sha256_file(base_path),
}
]
for candidate in sorted(base_path.rglob("*")):
if candidate.is_dir():
continue
relative = candidate.relative_to(base_path).as_posix()
if pattern and not fnmatch.fnmatch(relative, pattern):
continue
entries.append(
{
"path": relative,
"sizeBytes": candidate.stat().st_size,
"sha256": sha256_file(candidate),
}
)
return entries
def list_bos_entries(
bucket: str,
prefix: str,
pattern: str,
endpoint: str,
access_key_id: str,
secret_access_key: str,
) -> list[dict[str, Any]]:
client, scheme = build_bos_client(endpoint, access_key_id, secret_access_key)
normalized_prefix = bos_prefix(prefix)
prefix_with_slash = f"{normalized_prefix}/" if normalized_prefix else ""
entries: list[dict[str, Any]] = []
for candidate in client.list_all_objects(bucket, prefix=normalized_prefix or None):
key = text_value(getattr(candidate, "key", "")).strip().lstrip("/")
if not key or key.endswith("/"):
continue
if normalized_prefix:
if key == normalized_prefix:
continue
if not key.startswith(prefix_with_slash):
continue
relative_path = key[len(prefix_with_slash):]
else:
relative_path = key
if not relative_path:
continue
if pattern and not fnmatch.fnmatch(relative_path, pattern):
continue
size_bytes = int(getattr(candidate, "size", 0) or 0)
mime_type = mimetypes.guess_type(relative_path)[0] or ""
storage_path = canonical_bos_path(bucket, key)
entry: dict[str, Any] = {
"path": relative_path,
"storagePath": storage_path,
"sizeBytes": size_bytes,
"mimeType": mime_type,
"etag": text_value(getattr(candidate, "e_tag", "") or getattr(candidate, "etag", "")),
"versionId": text_value(getattr(candidate, "version_id", "") or getattr(candidate, "versionId", "")),
"lastModified": text_value(getattr(candidate, "last_modified", "") or getattr(candidate, "lastModified", "")),
"source": {"path": storage_path},
}
file_name = pathlib.PurePosixPath(relative_path).name.lower()
should_inline_json = relative_path.lower().endswith(".json") and (
file_name in XPLATFORM_INLINE_JSON_FILES or size_bytes <= PREVIEW_INLINE_BINARY_LIMIT
)
if should_inline_json:
raw = client.get_object_as_string(bucket, key)
payload_text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
try:
entry["data"] = json.loads(payload_text)
except Exception:
entry["data"] = {"value": payload_text}
elif pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
entry["source"]["previewURL"] = client.generate_pre_signed_url(
bucket,
key,
expiration_in_seconds=3600,
protocol=scheme,
)
entries.append(clean_metadata(entry) or {})
return entries
def source_entry_fingerprint(data_source: str, bucket: str, entry: dict[str, Any]) -> str:
normalized_source = (data_source or "BOS").strip().upper() or "BOS"
relative_path = str(entry.get("path", "")).strip()
version_id = str(entry.get("versionId", "")).strip()
sha256 = str(entry.get("sha256", "")).strip()
etag = str(entry.get("etag", "")).strip()
size_bytes = str(entry.get("sizeBytes", "")).strip()
last_modified = str(entry.get("lastModified", "")).strip()
parts = [normalized_source, str(bucket or "").strip(), relative_path]
if version_id:
parts.append(f"version:{version_id}")
elif sha256:
parts.append(f"sha256:{sha256}")
else:
parts.extend([f"etag:{etag}", f"size:{size_bytes}", f"lastModified:{last_modified}"])
raw = "\n".join(parts).encode("utf-8")
return hashlib.sha256(raw).hexdigest()
def incremental_unit(params: dict[str, str]) -> str:
value = (params.get("INCREMENTAL_UNIT") or "").strip().upper()
if not value:
return "OBJECT"
if value in {"OBJECT", "DIRECTORY"}:
return value
raise RuntimeError("INCREMENTAL_UNIT must be OBJECT or DIRECTORY")
def incremental_directory_depth(params: dict[str, str]) -> int:
return parse_positive_int_param(params, "INCREMENTAL_DIRECTORY_DEPTH", 1)
def directory_incremental_key(relative_path: str, depth: int) -> str:
clean_relative = str(relative_path or "").strip().lstrip("/")
if not clean_relative:
return "__root__"
parts = [segment for segment in pathlib.PurePosixPath(clean_relative).parts if segment not in ("", ".")]
if len(parts) <= 1:
return f"FILE::{clean_relative}"
directory_parts = parts[:-1]
take = min(max(depth, 1), len(directory_parts))
return "/".join(directory_parts[:take])
def incremental_source_state(params: dict[str, str]) -> dict[str, Any]:
raw = params.get("SOURCE_STATE_JSON", "").strip()
if not raw:
return {"seenFingerprints": []}
try:
parsed = json.loads(raw)
except Exception as exc:
raise RuntimeError("SOURCE_STATE_JSON must be valid JSON") from exc
if not isinstance(parsed, dict):
raise RuntimeError("SOURCE_STATE_JSON must be a JSON object")
seen = parsed.get("seenFingerprints")
if seen is None:
parsed["seenFingerprints"] = []
elif not isinstance(seen, list):
raise RuntimeError("SOURCE_STATE_JSON.seenFingerprints must be an array")
else:
normalized_seen = []
for index, item in enumerate(seen):
if not isinstance(item, str):
raise RuntimeError(f"SOURCE_STATE_JSON.seenFingerprints[{index}] must be a string")
fingerprint = item.strip()
if fingerprint:
normalized_seen.append(fingerprint)
parsed["seenFingerprints"] = normalized_seen
incremental_unit_value = str(parsed.get("incrementalUnit", "")).strip().upper()
if incremental_unit_value and incremental_unit_value not in {"DIRECTORY", "OBJECT"}:
raise RuntimeError("SOURCE_STATE_JSON.incrementalUnit must be OBJECT or DIRECTORY")
parsed["incrementalUnit"] = incremental_unit_value or "OBJECT"
directory_depth = parsed.get("directoryDepth", 1)
if isinstance(directory_depth, bool) or not isinstance(directory_depth, int):
raise RuntimeError("SOURCE_STATE_JSON.directoryDepth must be an integer")
if directory_depth <= 0:
raise RuntimeError("SOURCE_STATE_JSON.directoryDepth must be greater than 0")
parsed["directoryDepth"] = directory_depth
return parsed
def apply_incremental_source_filter(
entries: list[dict[str, Any]],
data_source: str,
bucket: str,
params: dict[str, str],
source_path: str,
prefix: str,
pattern: str,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
state = incremental_source_state(params)
seen = {str(item).strip() for item in state.get("seenFingerprints", []) if str(item).strip()}
scan_started_at = now_iso()
mode = incremental_unit(params)
depth = incremental_directory_depth(params)
emitted: list[dict[str, Any]] = []
updated_seen = set(seen)
if mode == "DIRECTORY":
grouped_entries: dict[str, list[dict[str, Any]]] = {}
for entry in entries:
group_key = directory_incremental_key(str(entry.get("path", "")).strip(), depth)
grouped_entries.setdefault(group_key, []).append(entry)
for group_key in sorted(grouped_entries.keys()):
group_entries = grouped_entries[group_key]
if group_key in updated_seen:
continue
updated_seen.add(group_key)
for entry in group_entries:
enriched = dict(entry)
enriched["fingerprint"] = source_entry_fingerprint(data_source, bucket, entry)
enriched["incrementalKey"] = group_key
emitted.append(enriched)
else:
for entry in entries:
fingerprint = source_entry_fingerprint(data_source, bucket, entry)
enriched = dict(entry)
enriched["fingerprint"] = fingerprint
enriched["incrementalKey"] = fingerprint
if fingerprint in updated_seen:
continue
updated_seen.add(fingerprint)
emitted.append(enriched)
scan_completed_at = now_iso()
checkpoint = {
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
"scanMode": "INCREMENTAL",
"incrementalUnit": mode,
"directoryDepth": depth,
"dataSource": data_source,
"sourcePath": source_path,
"bucket": bucket,
"prefix": prefix,
"pattern": pattern,
"seenFingerprints": sorted(updated_seen),
"listedCount": len(entries),
"newCount": len(emitted),
"scanStartedAt": scan_started_at,
"scanCompletedAt": scan_completed_at,
}
return emitted, checkpoint
def preview_payload_for_remote_entry(relative_path: str, mime_type: str, preview_url: str) -> dict[str, Any] | None:
normalized_preview_url = text_value(preview_url).strip()
if not normalized_preview_url:
return None
suffix = pathlib.PurePosixPath(relative_path).suffix.lower()
kind = "binary"
if suffix in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
kind = "video"
elif suffix in IMAGE_FILE_SUFFIXES or mime_type.startswith("image/"):
kind = "image"
elif suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
kind = "audio"
return {"preview": {"kind": kind, "format": suffix.lstrip(".") or kind, "url": normalized_preview_url}}
def source_entry_artifact(
run_id: str,
node_id: str,
entry: dict[str, Any],
source_path: str,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
relative_path = str(entry.get("path", "")).strip().lstrip("/")
source = manifest_source_ref(entry, relative_path, source_path)
storage_path = str(source.get("path", "")).strip() or source_path_join(source_path, relative_path)
mime_type = str(entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
artifact_type = infer_port_type(pathlib.Path(relative_path))
payload: dict[str, Any] | None = None
if "data" in entry:
payload = normalize_json_payload(entry.get("data"))
else:
preview_payload = preview_payload_for_remote_entry(relative_path, mime_type, str(source.get("previewURL", "")).strip())
payload = preview_payload if preview_payload else None
if payload is not None:
payload["relativePath"] = relative_path
if source:
payload["source"] = source
metadata = clean_metadata({"mimeType": mime_type, "sha256": entry.get("sha256", "")})
return build_virtual_artifact(
run_id,
node_id,
pathlib.PurePosixPath(relative_path).name or "artifact",
storage_path,
artifact_type,
int(entry.get("sizeBytes", 0) or 0),
data=payload,
metadata=metadata,
output=output,
)
def collect_output_artifacts(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
artifacts: list[dict[str, Any]] = []
for path in sorted(output_dir.rglob("*")):
if path.is_dir():
continue
artifact_type = infer_port_type(path)
mime_type, _ = mimetypes.guess_type(path.name)
if artifact_type == "json":
payload = load_payload_from_file(path)
else:
payload = extract_preview_data(path, artifact_type, mime_type or "")
binding = output
if binding is None and node_data is not None:
binding = output_binding(node_data)
artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding))
return artifacts
def hyperparams(node_data: dict[str, Any]) -> dict[str, str]:
values: dict[str, str] = {}
for item in node_data.get("hyperparams", []) or []:
key = str(item.get("key", "")).strip()
if not key:
continue
values[key] = str(item.get("value", ""))
return values
def parse_json_param(params: dict[str, str], key: str, default: Any, expected_type: type | tuple[type, ...] | None = None) -> Any:
raw = params.get(key, "").strip()
if not raw:
return default
try:
parsed = json.loads(raw)
except Exception as exc:
raise RuntimeError(f"{key} must be valid JSON") from exc
if expected_type is not None and not isinstance(parsed, expected_type):
type_name = "array" if expected_type is list else "object" if expected_type is dict else "expected type"
raise RuntimeError(f"{key} must be a JSON {type_name}")
return parsed
def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [load_artifact_payload(item) for item in inputs]
def collect_items(payloads: list[Any]) -> list[Any]:
items: list[Any] = []
for payload in payloads:
payload = normalize_json_payload(payload)
if isinstance(payload.get("items"), list):
items.extend(payload["items"])
elif isinstance(payload.get("entries"), list):
items.extend(payload["entries"])
elif isinstance(payload.get("rows"), list):
items.extend(payload["rows"])
else:
items.append(payload)
return items
def normalize_mongo_document(item: Any) -> dict[str, Any]:
if isinstance(item, dict):
if set(item.keys()) == {"value"} and isinstance(item.get("value"), dict):
return json_compatible(item["value"])
return json_compatible(item)
return {"value": json_compatible(item)}
def documents_from_inputs(payloads: list[dict[str, Any]], document_path: str) -> list[dict[str, Any]]:
raw_items: list[Any] = []
path = str(document_path or "").strip()
if path:
for payload in payloads:
value = get_field_value(payload, path)
if isinstance(value, list):
raw_items.extend(value)
elif value is not None:
raw_items.append(value)
else:
raw_items = collect_items(payloads)
return [normalize_mongo_document(item) for item in raw_items]
def build_mongo_client(uri: str):
try:
from pymongo import MongoClient
except Exception as exc:
raise RuntimeError("pymongo is required for MongoDB 写入算子") from exc
return MongoClient(uri, serverSelectionTimeoutMS=5000)
def get_field_value(value: Any, field_path: str) -> Any:
current = value
for token in [part for part in field_path.split(".") if part]:
if isinstance(current, dict):
current = current.get(token)
continue
if isinstance(current, list) and token.isdigit():
index = int(token)
if 0 <= index < len(current):
current = current[index]
continue
return None
return current
def normalize_comparison_operator(value: str, label: str = "OPERATOR", default: str = "eq") -> str:
operator = str(value or "").strip().lower() or default
if operator not in SUPPORTED_COMPARISON_OPERATORS:
supported = ", ".join(sorted(SUPPORTED_COMPARISON_OPERATORS))
raise RuntimeError(f"{label} must be one of {supported}")
return operator
def normalize_sampling_rules(rules: list[Any], label: str = "RULES_JSON") -> list[dict[str, Any]]:
normalized_rules: list[dict[str, Any]] = []
rule_ids: set[str] = set()
for index, rule in enumerate(rules):
if not isinstance(rule, dict):
raise RuntimeError(f"{label}[{index}] must be an object")
for key in ("id", "field", "operator", "samplePercent"):
if key not in rule:
raise RuntimeError(f"{label}[{index}].{key} is required")
rule_id = str(rule.get("id", "")).strip()
field_path = str(rule.get("field", "")).strip()
if not rule_id:
raise RuntimeError(f"{label}[{index}].id must be non-empty")
if rule_id in rule_ids:
raise RuntimeError(f"{label} contains duplicate id: {rule_id}")
rule_ids.add(rule_id)
if not field_path:
raise RuntimeError(f"{label}[{index}].field must be non-empty")
operator = normalize_comparison_operator(str(rule.get("operator", "")), f"{label}[{index}].operator")
if operator not in {"exists", "missing"} and "value" not in rule:
raise RuntimeError(f"{label}[{index}].value is required")
sample_percent = parse_sample_percent(rule.get("samplePercent"), f"{label}[{index}].samplePercent")
normalized_rules.append(
{
"id": rule_id,
"field": field_path,
"operator": operator,
"value": rule.get("value"),
"samplePercent": sample_percent,
}
)
return normalized_rules
def parse_sample_percent(value: Any, label: str) -> float:
if isinstance(value, bool):
raise RuntimeError(f"{label} must be numeric")
try:
sample_percent = float(value)
except Exception as exc:
raise RuntimeError(f"{label} must be numeric") from exc
if not math.isfinite(sample_percent):
raise RuntimeError(f"{label} must be finite")
if sample_percent <= 0 or sample_percent > 100:
raise RuntimeError(f"{label} must be in the range (0, 100]")
return sample_percent
def normalize_field_map(field_map: dict[Any, Any], label: str = "FIELD_MAP_JSON") -> dict[str, str]:
normalized: dict[str, str] = {}
destinations: set[str] = set()
for source, destination in field_map.items():
if not isinstance(source, str):
raise RuntimeError(f"{label} source field names must be strings")
source_name = source.strip()
if not source_name:
raise RuntimeError(f"{label} source field names must be non-empty")
if source_name != source:
raise RuntimeError(f"{label} source field {source!r} must not have leading or trailing whitespace")
if not isinstance(destination, str):
raise RuntimeError(f"{label}.{source_name} destination field must be a string")
destination_name = destination.strip()
if not destination_name:
raise RuntimeError(f"{label}.{source_name} destination field must be non-empty")
if destination_name != destination:
raise RuntimeError(f"{label}.{source_name} destination field must not have leading or trailing whitespace")
if destination_name in destinations:
raise RuntimeError(f"{label} maps multiple source fields to destination field {destination_name!r}")
destinations.add(destination_name)
normalized[source_name] = destination_name
return normalized
def map_json_row(row: dict[str, Any], field_map: dict[str, str], label: str = "FIELD_MAP_JSON") -> dict[str, Any]:
mapped: dict[str, Any] = {}
mapped_sources: dict[str, str] = {}
for key, value in row.items():
output_key = field_map.get(key, key)
if output_key in mapped:
previous_source = mapped_sources[output_key]
raise RuntimeError(f"{label} maps source fields {previous_source!r} and {key!r} to the same output field {output_key!r}")
mapped[output_key] = value
mapped_sources[output_key] = key
return mapped
def compare_values(actual: Any, operator: str, expected: Any) -> bool:
if operator == "exists":
return actual is not None
if operator == "missing":
return actual is None
if operator == "contains":
if isinstance(actual, list):
return expected in actual
return str(expected) in str(actual)
if operator == "in":
if isinstance(expected, list):
return actual in expected
return str(actual) in str(expected)
if operator in {"gt", "gte", "lt", "lte"}:
try:
actual_number = float(actual)
expected_number = float(expected)
except Exception:
return False
if operator == "gt":
return actual_number > expected_number
if operator == "gte":
return actual_number >= expected_number
if operator == "lt":
return actual_number < expected_number
return actual_number <= expected_number
if operator == "ne":
return actual != expected
return actual == expected
def flatten_context(value: Any, prefix: str = "") -> dict[str, str]:
context: dict[str, str] = {}
if isinstance(value, dict):
for key, item in value.items():
next_prefix = f"{prefix}.{key}" if prefix else str(key)
context.update(flatten_context(item, next_prefix))
if prefix:
context[prefix] = json.dumps(value, ensure_ascii=False)
return context
if isinstance(value, list):
for index, item in enumerate(value[:8]):
next_prefix = f"{prefix}.{index}" if prefix else str(index)
context.update(flatten_context(item, next_prefix))
if prefix:
context[prefix] = json.dumps(value, ensure_ascii=False)
return context
if prefix:
context[prefix] = "" if value is None else str(value)
return context
def render_template_text(template_text: str, context: dict[str, str]) -> str:
rendered = template_text
for key, value in context.items():
rendered = rendered.replace("{{" + key + "}}", value)
safe_context = {key: value for key, value in context.items()}
class SafeDict(dict):
def __missing__(self, key: str) -> str:
return "{" + key + "}"
try:
rendered = rendered.format_map(SafeDict(safe_context))
except Exception:
pass
return rendered
def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for item in collect_items(payloads):
if isinstance(item, dict):
rows.append(item)
else:
rows.append({"value": item})
return rows
def write_text_artifact(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
filename: str,
content: str,
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
path = output_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
mime_type, _ = mimetypes.guess_type(path.name)
preview = extract_preview_data(path, infer_port_type(path), mime_type or "")
return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output)
def write_csv_artifact(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
filename: str,
rows: list[dict[str, Any]],
output: dict[str, Any] | None = None,
) -> dict[str, Any]:
path = output_dir / filename
path.parent.mkdir(parents=True, exist_ok=True)
columns: list[str] = []
for row in rows:
for key in row.keys():
if key not in columns:
columns.append(key)
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=columns or ["value"])
writer.writeheader()
if rows:
writer.writerows(rows)
preview = preview_tabular_payload(path, ",", "csv")
return build_artifact(run_id, node_id, path, "file", data=preview, output=output)
def write_video_segment(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
cv2, _ = require_cv2()
capture = cv2.VideoCapture(str(source_path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频文件: {source_path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
start_frame = max(0, int(start_second * fps))
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
if frame_count > 0:
end_frame = min(end_frame, frame_count)
capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
destination.parent.mkdir(parents=True, exist_ok=True)
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
if not writer.isOpened():
raise RuntimeError(f"无法创建视频切片: {destination}")
written = 0
try:
for _ in range(start_frame, end_frame):
ok, frame = capture.read()
if not ok:
break
writer.write(frame)
written += 1
finally:
writer.release()
duration = written / fps if fps > 0 else 0.0
return {
"frameRate": round(fps, 3),
"frameCount": written,
"resolution": f"{width}x{height}",
"duration": round(duration, 3),
}
finally:
capture.release()
def ffmpeg_available() -> bool:
return shutil.which("ffmpeg") is not None
def run_ffmpeg(args: list[str]) -> None:
completed = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if completed.returncode == 0:
return
stderr = (completed.stderr or "").strip()
stdout = (completed.stdout or "").strip()
message = stderr or stdout or "unknown ffmpeg error"
raise RuntimeError(f"ffmpeg execution failed: {message}")
def write_video_segment_with_ffmpeg(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
if not ffmpeg_available():
raise RuntimeError("ffmpeg is not available in operator-runner image")
duration = max(float(end_second) - float(start_second), 0.001)
destination.parent.mkdir(parents=True, exist_ok=True)
copy_args = [
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
"-ss",
f"{max(float(start_second), 0.0):.3f}",
"-i",
str(source_path),
"-t",
f"{duration:.3f}",
"-map",
"0:v:0",
"-map",
"0:a?",
"-c",
"copy",
str(destination),
]
reencode_args = [
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
"-ss",
f"{max(float(start_second), 0.0):.3f}",
"-i",
str(source_path),
"-t",
f"{duration:.3f}",
"-map",
"0:v:0",
"-map",
"0:a?",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"23",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
str(destination),
]
try:
run_ffmpeg(copy_args)
except RuntimeError:
run_ffmpeg(reencode_args)
metrics = inspect_video_file(destination)
return {
"frameRate": metrics.get("frameRate"),
"frameCount": metrics.get("frameCount"),
"resolution": metrics.get("resolution"),
"duration": metrics.get("duration", round(duration, 3)),
}
def write_video_segments(
source_path: pathlib.Path | str,
segments: list[dict[str, Any]],
) -> list[dict[str, Any]]:
cv2, _ = require_cv2()
capture = cv2.VideoCapture(str(source_path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频文件: {source_path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
normalized_segments: list[dict[str, Any]] = []
for raw in segments:
destination = pathlib.Path(raw.get("destination", ""))
start_second = float(raw.get("startSecond") or 0.0)
end_second = float(raw.get("endSecond") or 0.0)
start_frame = max(0, int(start_second * fps))
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
if frame_count > 0:
end_frame = min(end_frame, frame_count)
normalized_segments.append(
{
**raw,
"destination": destination,
"startFrame": start_frame,
"endFrame": end_frame,
}
)
normalized_segments.sort(key=lambda item: (int(item["startFrame"]), str(item["destination"])))
current_frame = 0
if normalized_segments:
initial_frame = int(normalized_segments[0]["startFrame"])
if initial_frame > 0:
capture.set(cv2.CAP_PROP_POS_FRAMES, initial_frame)
current_frame = initial_frame
results: list[dict[str, Any]] = []
for segment in normalized_segments:
start_frame = int(segment["startFrame"])
end_frame = int(segment["endFrame"])
destination = pathlib.Path(segment["destination"])
if current_frame < start_frame:
while current_frame < start_frame:
ok, _ = capture.read()
if not ok:
current_frame = end_frame
break
current_frame += 1
destination.parent.mkdir(parents=True, exist_ok=True)
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
if not writer.isOpened():
raise RuntimeError(f"无法创建视频切片: {destination}")
written = 0
try:
while current_frame < end_frame:
ok, frame = capture.read()
if not ok:
break
writer.write(frame)
written += 1
current_frame += 1
finally:
writer.release()
results.append(
{
**segment,
"frameRate": round(fps, 3),
"frameCount": written,
"resolution": f"{width}x{height}",
"duration": round(written / fps, 3) if fps > 0 else 0.0,
}
)
return results
finally:
capture.release()
def write_video_segment_from_remote_url(source_url: str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
return write_video_segment(source_url, destination, start_second, end_second)
def annotate_video_clip(path: pathlib.Path, frame_interval_seconds: float) -> dict[str, Any]:
cv2, np = require_cv2()
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise RuntimeError(f"无法打开视频切片: {path}")
try:
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
step = max(1, int(round(max(frame_interval_seconds, 0.1) * fps)))
previous_gray = None
samples: list[tuple[float, float]] = []
objects: set[str] = set()
frame_index = 0
while True:
ok, frame = capture.read()
if not ok:
break
if frame_index % step != 0:
frame_index += 1
continue
timestamp = frame_index / fps if fps > 0 else 0.0
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
motion = 0.0
if previous_gray is not None:
diff = cv2.absdiff(previous_gray, gray)
motion = float(diff.mean() / 255.0)
previous_gray = gray
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
mask = cv2.inRange(ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8))
if float(mask.mean() / 255.0) > 0.1:
objects.add("手部")
_, threshold = cv2.threshold(gray, 24, 255, cv2.THRESH_BINARY)
contours, _ = cv2.findContours(threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
if any(cv2.contourArea(contour) >= 64 for contour in contours):
objects.add("前景目标")
samples.append((timestamp, motion))
frame_index += 1
if not samples:
samples.append((0.0, 0.0))
def label_for_motion(motion: float) -> tuple[str, float]:
if motion < 0.015:
return "静止观察", 0.88
if motion < 0.045:
return "平稳活动", 0.79
return "快速移动", 0.71
actions: list[dict[str, Any]] = []
last_timestamp = duration
for timestamp, motion in samples:
label, confidence = label_for_motion(motion)
last_timestamp = timestamp
if not actions:
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
continue
current = actions[-1]
if current["label"] != label:
current["end"] = round(timestamp, 3)
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
else:
current["end"] = round(timestamp, 3)
current["confidence"] = round(min(float(current["confidence"]), confidence), 3)
if actions:
actions[-1]["end"] = round(max(duration, last_timestamp), 3)
min_confidence = min(float(item["confidence"]) for item in actions) if actions else 0.0
return {
"duration": round(duration, 3),
"actions": actions,
"objects": sorted(objects),
"minConfidence": round(min_confidence, 3),
"frameIntervalSeconds": frame_interval_seconds,
}
finally:
capture.release()
def builtin_source(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT"))
data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper()
if data_source and data_source not in VALID_SOURCE_DATA_SOURCES:
raise RuntimeError("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
source_path = (params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "")).strip()
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
scan_mode = (params.get("SCAN_MODE") or "").strip().upper()
if scan_mode and scan_mode not in {"FULL", "INCREMENTAL"}:
raise RuntimeError("SCAN_MODE must be FULL or INCREMENTAL")
unit = incremental_unit(params)
depth = incremental_directory_depth(params)
source_bucket, source_prefix = parse_bos_path(source_path)
if not bucket and source_bucket:
bucket = source_bucket
if not prefix and source_prefix:
prefix = source_prefix
local_source_path = pathlib.Path(source_path) if source_path and not source_bucket else None
local_prefix_path = pathlib.Path(prefix) if prefix and "://" not in prefix else None
has_bos_connection = bool(bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key)
if inline_source:
data_source = "INLINE"
elif not data_source:
if source_bucket or has_bos_connection:
data_source = "BOS"
elif local_source_path is not None:
data_source = "LOCAL"
elif local_prefix_path is not None and local_prefix_path.exists():
data_source = "LOCAL"
elif inputs:
data_source = "UPSTREAM"
if data_source == "INLINE" and not inline_source:
raise RuntimeError("真实数据源 DATA_SOURCE=INLINE requires SOURCE_TEXT or SOURCE_JSON")
if data_source == "BOS":
missing = []
if not bucket:
missing.append("bucket or bos:// SOURCE_PATH")
if not bos_endpoint:
missing.append("BOS_ENDPOINT")
if not bos_access_key_id:
missing.append("BOS_ACCESS_KEY_ID")
if not bos_secret_access_key:
missing.append("BOS_SECRET_ACCESS_KEY")
if missing:
raise RuntimeError("真实数据源 DATA_SOURCE=BOS requires " + ", ".join(missing))
if data_source == "LOCAL":
if source_bucket:
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL cannot read a bos:// SOURCE_PATH")
if local_source_path is not None and not local_source_path.exists():
raise RuntimeError(f"真实数据源 SOURCE_PATH does not exist inside operator-runner: {source_path}")
if local_source_path is None and (local_prefix_path is None or not local_prefix_path.exists()):
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL requires SOURCE_PATH or an existing STORAGE_PREFIX path")
if not data_source:
raise RuntimeError(
"真实数据源 requires SOURCE_TEXT, SOURCE_JSON, SOURCE_PATH, a configured BOS source, "
"an existing local STORAGE_PREFIX path, or upstream input artifacts"
)
entries: list[dict[str, Any]] = []
resolved_source_path = source_path
if params.get("SOURCE_JSON"):
parsed = json.loads(params["SOURCE_JSON"])
if isinstance(parsed, list):
entries = [{"value": item} for item in parsed]
else:
entries = [{"value": parsed}]
elif params.get("SOURCE_TEXT"):
entries = [{"text": params["SOURCE_TEXT"]}]
elif data_source == "BOS":
cleaned_prefix = bos_prefix(prefix)
entries = list_bos_entries(
bucket,
cleaned_prefix,
pattern,
bos_endpoint,
bos_access_key_id,
bos_secret_access_key,
)
resolved_source_path = canonical_bos_path(bucket, cleaned_prefix)
elif local_source_path is not None:
entries = list_source_entries(local_source_path, pattern)
elif local_prefix_path is not None and local_prefix_path.exists():
entries = list_source_entries(local_prefix_path, pattern)
elif inputs:
entries = [load_artifact_payload(item) for item in inputs]
listed_entries = list(entries)
checkpoint_payload = None
if scan_mode == "INCREMENTAL":
entries, checkpoint_payload = apply_incremental_source_filter(
listed_entries,
data_source,
bucket,
params,
resolved_source_path,
prefix,
pattern,
)
payload = {
"dataSource": data_source,
"bucket": bucket,
"prefix": prefix,
"endpoint": bos_endpoint,
"pattern": pattern,
"scanMode": scan_mode or "FULL",
"incrementalUnit": unit,
"directoryDepth": depth,
"sourcePath": resolved_source_path,
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
"listedCount": len(listed_entries),
"newCount": len(entries) if checkpoint_payload is not None else len(entries),
"entries": entries,
"catalogEntries": listed_entries,
}
artifacts = [
write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))
]
if checkpoint_payload is not None:
artifacts.append(write_json_artifact(run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload))
if data_source == "BOS":
for entry in entries:
artifacts.append(source_entry_artifact(run_id, node_id, entry, resolved_source_path, output=output_binding(node_data)))
elif source_path and pathlib.Path(source_path).exists():
source_root = pathlib.Path(source_path)
staged_root = output_dir / "_source_tree"
for entry in entries:
relative_path = str(entry.get("path", "")).strip()
if not relative_path:
continue
candidate = source_root / relative_path if source_root.is_dir() else source_root
if not candidate.exists() or not candidate.is_file():
continue
destination = staged_root / relative_path
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(candidate, destination)
artifacts.extend(collect_output_artifacts(run_id, node_id, staged_root, node_data))
return artifacts
def builtin_merge(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = payloads_from_inputs(inputs)
merged = merge_dicts(payloads)
merged["inputCount"] = len(payloads)
return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))]
def builtin_intersection(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = payloads_from_inputs(inputs)
if not payloads:
return [
write_json_artifact(
run_id,
node_id,
output_dir,
"intersection.json",
{"keys": [], "inputCount": 0},
output=output_binding(node_data),
)
]
key_sets = [set(payload.keys()) for payload in payloads]
common_keys = sorted(set.intersection(*key_sets)) if key_sets else []
payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}}
return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))]
def builtin_distribution(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = payloads_from_inputs(inputs)
items = collect_items(payloads)
outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}]
partitions: list[list[Any]] = [[] for _ in outputs]
for index, item in enumerate(items):
partitions[index % len(outputs)].append(item)
artifacts: list[dict[str, Any]] = []
for index, output in enumerate(outputs):
filename = f"{output.get('name', f'queue-{index}')}.json"
payload = {"items": partitions[index], "count": len(partitions[index])}
artifacts.append(
write_json_artifact(
run_id,
node_id,
output_dir,
filename,
payload,
str(output.get("type", "json")),
output=output,
)
)
return artifacts
def builtin_upload(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip()
bucket = str(node_data.get("bucket", "")).strip()
prefix = str(node_data.get("prefix", "")).strip()
parsed_bucket, parsed_prefix = parse_bos_path(upload_path)
if parsed_bucket:
bucket = parsed_bucket
if parsed_prefix or upload_path.endswith("/"):
prefix = parsed_prefix
data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper()
bos_endpoint = params.get("BOS_ENDPOINT", "").strip()
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip()
copied_paths: list[str] = []
bundle_inputs: list[dict[str, Any]] = []
staging_dir = output_dir / "uploaded"
staging_dir.mkdir(parents=True, exist_ok=True)
package_manifest_payload = first_payload_with_keys(inputs, "packageCount", "items")
upload_inputs = list(inputs)
if package_manifest_payload is not None:
package_names = package_related_artifact_names(package_manifest_payload)
upload_inputs = [
artifact
for artifact in inputs
if str(artifact.get("name", "")).strip() not in package_names
]
upload_inputs.extend(
materialize_package_artifacts_from_manifest(
run_id,
node_id,
staging_dir,
package_manifest_payload,
node_data,
)
)
if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key:
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
temp_paths = [
str(item.get("path", "")).strip()
for item in upload_inputs
if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip()
]
input_root = common_input_root(temp_paths)
uploaded_objects: list[dict[str, Any]] = []
remote_artifacts: list[dict[str, Any]] = []
for index, artifact in enumerate(upload_inputs):
payload = load_artifact_payload(artifact)
bundle_inputs.append(payload)
fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}"
relative_path = upload_relative_path(artifact, input_root, fallback_name)
temp_source = materialized_upload_source(artifact, staging_dir, relative_path)
if temp_source is None or not temp_source.exists():
continue
mime_type = (
str((artifact.get("metadata") or {}).get("mimeType", "")).strip()
if isinstance(artifact.get("metadata"), dict)
else ""
) or mimetypes.guess_type(temp_source.name)[0] or ""
object_key = bos_key_join(prefix, relative_path)
put_kwargs = {"content_type": mime_type} if mime_type else {}
try:
size_bytes = temp_source.stat().st_size
except OSError:
size_bytes = 0
log(
f"[operator-runner][info] upload starts: {temp_source} -> {canonical_bos_path(bucket, object_key)} "
f"({size_bytes} bytes)"
)
client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs)
remote_path = canonical_bos_path(bucket, object_key)
log(
f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} "
f"({size_bytes} bytes)"
)
preview_url = ""
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
preview_url = text_value(
client.generate_pre_signed_url(
bucket,
object_key,
expiration_in_seconds=3600,
protocol=scheme,
)
).strip()
uploaded_objects.append(
clean_metadata(
{
"name": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": remote_path,
"previewURL": preview_url,
"sizeBytes": temp_source.stat().st_size,
"mimeType": mime_type,
}
)
or {}
)
artifact_type = str(artifact.get("type", "")).strip() or infer_port_type(temp_source)
artifact_payload: dict[str, Any] | None = None
if artifact_type == "json":
loaded = load_payload_from_file(temp_source)
if loaded is not None:
artifact_payload = normalize_json_payload(loaded)
else:
artifact_payload = preview_payload_for_remote_entry(relative_path, mime_type, preview_url)
if artifact_payload is None:
artifact_payload = extract_preview_data(temp_source, artifact_type, mime_type)
if artifact_payload is not None:
artifact_payload["relativePath"] = relative_path
artifact_payload["source"] = clean_metadata(
{
"path": remote_path,
"previewURL": preview_url,
"relativePath": relative_path,
"mimeType": mime_type,
}
) or {}
source_size_bytes = temp_source.stat().st_size
source_sha256, source_sha256_metadata = artifact_sha256(temp_source, source_size_bytes)
remote_artifacts.append(
build_virtual_artifact(
run_id,
node_id,
pathlib.PurePosixPath(relative_path).name,
remote_path,
artifact_type,
source_size_bytes,
data=artifact_payload,
metadata={
**source_sha256_metadata,
"sha256": source_sha256,
"mimeType": mime_type,
},
output=output_binding(node_data),
)
)
payload = {
"copiedPaths": copied_paths,
"uploadedCount": len(uploaded_objects),
"uploadedObjects": uploaded_objects,
"items": bundle_inputs,
"inputCount": len(inputs),
"dataSource": "BOS",
"bucket": bucket,
"prefix": prefix,
"destinationPath": canonical_bos_path(bucket, prefix),
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
artifacts.extend(remote_artifacts)
return artifacts
for artifact in upload_inputs:
payload = load_artifact_payload(artifact)
bundle_inputs.append(payload)
source_path = artifact.get("path")
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
source = pathlib.Path(source_path)
destination = staging_dir / source.name
if source.resolve() != destination.resolve():
shutil.copy2(source, destination)
copied_paths.append(str(destination))
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir))
return artifacts
def builtin_json_extract(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_path = params.get("FIELD_PATH", "").strip()
payloads = payloads_from_inputs(inputs)
payload = payloads[0] if payloads else {}
extracted = get_field_value(payload, field_path) if field_path else payload
result = {"fieldPath": field_path, "value": extracted}
return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))]
def builtin_json_filter(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_path = params.get("FIELD_PATH", "").strip()
operator = normalize_comparison_operator(params.get("OPERATOR", "eq"))
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
rows = collect_items(payloads_from_inputs(inputs))
filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
payload = {
"fieldPath": field_path,
"operator": operator,
"expected": expected,
"items": filtered,
"count": len(filtered),
}
return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))]
def builtin_json_map(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_map = normalize_field_map(parse_json_param(params, "FIELD_MAP_JSON", {}, dict))
rows = rows_from_payloads(payloads_from_inputs(inputs))
mapped = [map_json_row(row, field_map) for row in rows]
payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)}
return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))]
def builtin_mongodb_write(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
mongo_target = params.get("MONGO_TARGET", "").strip().lower() or "business"
explicit_mongo_uri = params.get("MONGO_URI", "").strip() or os.getenv("MONGO_URI", "").strip()
explicit_mongo_db = params.get("MONGO_DB", "").strip() or os.getenv("MONGO_DB", "").strip()
collection_name = params.get("MONGO_COLLECTION", "").strip()
mode = params.get("MONGO_MODE", "insert_many").strip().lower() or "insert_many"
upsert_key = params.get("UPSERT_KEY", "_id").strip() or "_id"
document_path = params.get("DOCUMENT_PATH", "").strip()
mongo_uri = explicit_mongo_uri
mongo_db = explicit_mongo_db
if not mongo_uri:
if mongo_target == "platform":
mongo_uri = os.getenv("PLATFORM_MONGO_URI", "").strip()
mongo_db = mongo_db or os.getenv("PLATFORM_MONGO_DB", "").strip()
else:
mongo_uri = os.getenv("BUSINESS_MONGO_URI", "").strip()
mongo_db = mongo_db or os.getenv("BUSINESS_MONGO_DB", "").strip()
if not mongo_uri:
raise RuntimeError(f"MongoDB 写入缺少可用连接:target={mongo_target}")
if not mongo_db:
raise RuntimeError(f"MongoDB 写入缺少数据库名:target={mongo_target}")
if not collection_name:
raise RuntimeError("MONGO_COLLECTION is required for MongoDB 写入")
if mongo_target == "platform" and collection_name.lower() in {item.lower() for item in RESERVED_MONGO_COLLECTIONS}:
raise RuntimeError(f"collection {collection_name} is reserved and cannot be written by workflow operators")
if mode not in {"insert_many", "upsert"}:
raise RuntimeError("MONGO_MODE must be insert_many or upsert")
documents = documents_from_inputs(payloads_from_inputs(inputs), document_path)
if not documents:
raise RuntimeError("MongoDB 写入未收到可写入的文档")
client = build_mongo_client(mongo_uri)
try:
collection = client[mongo_db][collection_name]
summary: dict[str, Any] = {
"target": mongo_target,
"database": mongo_db,
"collection": collection_name,
"mode": mode,
"documentCount": len(documents),
}
if document_path:
summary["documentPath"] = document_path
if mode == "insert_many":
result = collection.insert_many(documents, ordered=False)
summary["insertedCount"] = len(getattr(result, "inserted_ids", []) or [])
else:
processed = 0
matched_count = 0
modified_count = 0
upserted_ids: list[Any] = []
for document in documents:
key_value = get_field_value(document, upsert_key)
if key_value is None:
raise RuntimeError(f"document is missing UPSERT_KEY={upsert_key}")
result = collection.replace_one({upsert_key: key_value}, document, upsert=True)
processed += 1
matched_count += int(getattr(result, "matched_count", 0) or 0)
modified_count += int(getattr(result, "modified_count", 0) or 0)
upserted_id = getattr(result, "upserted_id", None)
if upserted_id is not None:
upserted_ids.append(json_compatible(upserted_id))
summary["upsertKey"] = upsert_key
summary["upsertedCount"] = processed
summary["matchedCount"] = matched_count
summary["modifiedCount"] = modified_count
if upserted_ids:
summary["upsertedIds"] = upserted_ids
return [
write_json_artifact(
run_id,
node_id,
output_dir,
"mongodb-write-result.json",
summary,
output=output_binding(node_data),
)
]
finally:
close = getattr(client, "close", None)
if callable(close):
close()
def builtin_text_template(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
template_text = params.get("TEMPLATE_TEXT", "").strip()
output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt"
payloads = payloads_from_inputs(inputs)
context = flatten_context(payloads[0] if payloads else {})
rendered = render_template_text(template_text, context)
binding = output_binding(node_data)
manifest = write_json_artifact(
run_id,
node_id,
output_dir,
"rendered-template.json",
{"template": template_text, "context": context, "outputName": output_name},
output=binding,
)
return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)]
def builtin_csv_to_json(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
payload: dict[str, Any]
if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES:
delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()]
columns, rows = load_tabular_rows(source_path, delimiter)
payload = {
"sourcePath": str(source_path),
"columns": columns,
"items": rows,
"rowCount": len(rows),
}
else:
for artifact in inputs:
preview = artifact_preview_payload(artifact) or {}
if preview.get("kind") == "table":
payload = {
"sourcePath": str(artifact.get("path", "")),
"columns": preview.get("columns", []),
"items": preview.get("rows", []),
"rowCount": preview.get("rowCount", len(preview.get("rows", []))),
"previewTruncated": preview.get("truncated", False),
}
return [
write_json_artifact(
run_id,
node_id,
output_dir,
"table.json",
payload,
output=output_binding(node_data),
)
]
raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件")
return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))]
def builtin_json_to_csv(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
rows = rows_from_payloads(payloads_from_inputs(inputs))
binding = output_binding(node_data)
summary = write_json_artifact(
run_id,
node_id,
output_dir,
"csv-export-summary.json",
{"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})},
output=binding,
)
return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)]
def builtin_directory_scan(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
if source_path is not None:
root = source_path if source_path.is_dir() else source_path.parent
payload = {
"sourcePath": str(source_path),
"scanRoot": str(root),
"summary": summarize_directory_tree(root),
}
elif inputs:
payload = {
"sourcePath": "",
"scanRoot": "",
"summary": summarize_input_artifacts(inputs),
}
else:
raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录")
return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))]
def builtin_xplatform_index(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payload = load_xplatform_index_payload(hyperparams(node_data), inputs, output_dir)
return [write_json_artifact(run_id, node_id, output_dir, "dataset-index.json", payload, output=output_binding(node_data))]
def builtin_collection_validate(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
require_intrinsics = parse_bool_param(params, "REQUIRE_INTRINSICS", False)
payload = first_payload_with_keys(inputs, "collections", "items")
if payload is None:
payload = load_xplatform_index_payload(params, inputs, output_dir)
collections = []
valid_collection_count = 0
for collection in payload.get("collections", []):
if not isinstance(collection, dict):
continue
missing_required = list(collection.get("missingRequiredFiles", []))
if require_intrinsics and not collection.get("hasIntrinsics", False):
missing_required.append("intrinsics.json")
valid = len(missing_required) == 0 and int(collection.get("videoCount", 0) or 0) > 0
if valid:
valid_collection_count += 1
collections.append({**collection, "missingRequiredFiles": missing_required, "valid": valid})
items = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
collection_id = str(item.get("collectionId", "")).strip()
items.append({**item, "collectionValid": any(c.get("id") == collection_id and c.get("valid", False) for c in collections)})
result = {
**payload,
"datasetType": "x-platform-validation",
"collections": collections,
"items": items,
"validCollectionCount": valid_collection_count,
"invalidCollectionCount": max(len(collections) - valid_collection_count, 0),
}
return [write_json_artifact(run_id, node_id, output_dir, "dataset-validation.json", result, output=output_binding(node_data))]
def builtin_video_quality(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
min_clarity = parse_unit_interval_param(params, "MIN_CLARITY", DEFAULT_MIN_CLARITY)
max_shake = parse_unit_interval_param(params, "MAX_SHAKE", DEFAULT_MAX_SHAKE)
max_hand_occlusion = parse_unit_interval_param(params, "MAX_HAND_OCCLUSION", DEFAULT_MAX_HAND_OCCLUSION)
clarity_scale = parse_positive_float_param(params, "CLARITY_SCALE", 35.0)
payload = first_payload_with_keys(inputs, "items")
if payload is None:
payload = load_xplatform_index_payload(params, inputs, output_dir)
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
items = []
passed_count = 0
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
source_path = source_path_from_item(item, str(item.get("filename", "video.mp4")), output_dir, archive_sources)
metrics = analyze_video_quality_metrics(source_path, clarity_scale)
passed = (
float_value(metrics.get("clarity"), 0.0) >= min_clarity
and float_value(metrics.get("shake"), 1.0) <= max_shake
and float_value(metrics.get("handOcclusion"), 1.0) <= max_hand_occlusion
)
if passed:
passed_count += 1
items.append(
{
**item,
"quality": {
**metrics,
"passed": passed,
"thresholds": {
"minClarity": min_clarity,
"maxShake": max_shake,
"maxHandOcclusion": max_hand_occlusion,
},
},
}
)
result = {
**payload,
"datasetType": "x-platform-quality",
"items": items,
"videoCount": len(items),
"passedCount": passed_count,
"failedCount": max(len(items) - passed_count, 0),
}
return [write_json_artifact(run_id, node_id, output_dir, "quality-report.json", result, output=output_binding(node_data))]
def builtin_dataset_filter(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "items")
if payload is None:
raise RuntimeError("数据过滤需要上游数据清单输入")
date_start = normalize_date_value(params.get("DATE_START", ""))
date_end = normalize_date_value(params.get("DATE_END", ""))
place = params.get("PLACE", "").strip()
collection_type = params.get("TYPE", "").strip()
min_clarity = parse_unit_interval_param(params, "MIN_CLARITY", DEFAULT_MIN_CLARITY)
max_shake = parse_unit_interval_param(params, "MAX_SHAKE", DEFAULT_MAX_SHAKE)
max_hand_occlusion = parse_unit_interval_param(params, "MAX_HAND_OCCLUSION", DEFAULT_MAX_HAND_OCCLUSION)
filtered = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
meta = item.get("meta") if isinstance(item.get("meta"), dict) else {}
quality = item.get("quality") if isinstance(item.get("quality"), dict) else {}
if place and str(meta.get("place", "")).strip() != place:
continue
if collection_type and str(meta.get("type", "")).strip() != collection_type:
continue
if date_start or date_end:
if not date_in_range(meta.get("date", ""), date_start, date_end):
continue
if float_value(quality.get("clarity"), 0.0) < min_clarity:
continue
if float_value(quality.get("shake"), 1.0) > max_shake:
continue
if float_value(quality.get("handOcclusion"), 1.0) > max_hand_occlusion:
continue
if not bool(quality.get("passed", False)):
continue
filtered.append(item)
result = {
"datasetType": "x-platform-filtered",
"criteria": {
"dateStart": date_start,
"dateEnd": date_end,
"place": place,
"type": collection_type,
"minClarity": min_clarity,
"maxShake": max_shake,
"maxHandOcclusion": max_hand_occlusion,
},
"count": len(filtered),
"items": filtered,
}
if isinstance(payload.get("archiveSources"), dict):
result["archiveSources"] = payload["archiveSources"]
return [write_json_artifact(run_id, node_id, output_dir, "filtered-videos.json", result, output=output_binding(node_data))]
def builtin_video_slicing(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "items")
if payload is None:
raise RuntimeError("视频切片需要上游过滤结果输入")
segment_duration = parse_positive_int_param(params, "SEGMENT_DURATION_SECONDS", 300)
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
clips: list[dict[str, Any]] = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
source_filename = str(item.get("filename", "")).strip() or "video.mp4"
source_path = source_path_from_item(item, source_filename, output_dir, archive_sources)
metrics = inspect_video_file(source_path)
duration = float(item.get("duration") or metrics.get("duration") or 0.0)
if duration <= 0:
duration = float(metrics.get("duration") or 0.0)
if duration <= 0:
duration = float(segment_duration)
clip_count = max(1, int(math.ceil(duration / segment_duration)))
stem = pathlib.Path(source_filename).stem
clip_specs: list[dict[str, Any]] = []
for clip_index in range(clip_count):
clip_start = clip_index * segment_duration
clip_end = min(duration, clip_start + segment_duration)
clip_name = f"{stem}_clip{clip_index + 1:03d}.mp4"
clip_path = output_dir / "clips" / clip_name
clip_specs.append(
{
"clipIndex": clip_index,
"clipStart": clip_start,
"clipEnd": clip_end,
"clipName": clip_name,
"clipPath": clip_path,
"destination": clip_path,
"startSecond": clip_start,
"endSecond": clip_end,
}
)
clip_metrics_list: list[dict[str, Any]] | None = None
if ffmpeg_available():
try:
log(f"[operator-runner][info] video_slicing uses ffmpeg for {source_filename} ({clip_count} clip(s))")
clip_metrics_list = []
for clip_spec in clip_specs:
clip_metrics_list.append(
{
**clip_spec,
**write_video_segment_with_ffmpeg(
source_path,
pathlib.Path(clip_spec["clipPath"]),
float(clip_spec["clipStart"]),
float(clip_spec["clipEnd"]),
),
}
)
except RuntimeError as exc:
log(f"[operator-runner][warn] ffmpeg video slicing failed for {source_filename}, fallback to opencv: {exc}")
clip_metrics_list = None
if clip_metrics_list is None:
if len(clip_specs) > 1:
clip_metrics_list = write_video_segments(source_path, clip_specs)
else:
only = clip_specs[0]
clip_metrics_list = [
{
**only,
**write_video_segment(source_path, only["clipPath"], float(only["clipStart"]), float(only["clipEnd"])),
}
]
for clip_spec in clip_metrics_list:
clip_start = float(clip_spec["clipStart"])
clip_end = float(clip_spec["clipEnd"])
clip_name = str(clip_spec["clipName"])
clip_path = pathlib.Path(clip_spec["clipPath"])
clips.append(
{
"clipId": pathlib.Path(clip_name).stem,
"clipName": clip_name,
"clipPath": str(clip_path),
"sourceVideo": source_filename,
"sourcePath": str(source_path),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
"collectionId": item.get("collectionId", ""),
"meta": item.get("meta", {}),
"quality": item.get("quality", {}),
"startSecond": round(clip_start, 3),
"endSecond": round(clip_end, 3),
"duration": clip_spec.get("duration", round(max(clip_end - clip_start, 0.0), 3)),
"resolution": clip_spec.get("resolution"),
"frameRate": clip_spec.get("frameRate"),
}
)
manifest = {
"datasetType": "x-platform-clips",
"segmentDurationSeconds": segment_duration,
"clipCount": len(clips),
"items": clips,
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "clips-manifest.json", manifest, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "clips", node_data=node_data, output=output_binding(node_data)))
return artifacts
def builtin_video_annotation(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "clipCount", "items")
if payload is None:
raise RuntimeError("视频标注需要上游切片结果输入")
frame_interval = parse_positive_float_param(params, "FRAME_INTERVAL_SECONDS", 1.0)
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
items = []
for clip in payload.get("items", []):
if not isinstance(clip, dict):
continue
clip_name = str(clip.get("clipName", "")).strip() or pathlib.Path(str(clip.get("clipPath", ""))).name
artifact = clip_artifacts.get(clip_name)
clip_path: pathlib.Path | None = None
if artifact is not None:
try:
clip_path = source_path_from_item(
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
clip_name,
output_dir,
archive_sources,
)
except RuntimeError:
clip_path = None
if clip_path is None:
direct_clip_path = str(clip.get("clipPath", "")).strip()
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
clip_path = pathlib.Path(direct_clip_path)
if clip_path is None:
clip_path = materialize_clip_from_source(clip, output_dir, archive_sources)
if clip_path is None or not clip_path.exists():
raise RuntimeError(f"无法获取真实输入文件: {clip_name}")
annotation = annotate_video_clip(clip_path, frame_interval)
annotation_filename = f"{pathlib.Path(clip_name).stem}_annotation.json"
annotation_payload = {
"clipId": clip.get("clipId", pathlib.Path(clip_name).stem),
"sourceVideo": clip.get("sourceVideo", ""),
**annotation,
}
items.append({**clip, "annotation": annotation_payload, "annotationFile": annotation_filename})
write_json_artifact(
run_id,
node_id,
output_dir / "annotations",
annotation_filename,
annotation_payload,
output=output_binding(node_data),
)
manifest = {
"datasetType": "x-platform-annotations",
"annotationCount": len(items),
"items": items,
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "annotations-manifest.json", manifest, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "annotations", node_data=node_data, output=output_binding(node_data)))
return artifacts
def builtin_result_package(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payload = first_payload_with_keys(inputs, "annotationCount", "items")
if payload is None:
payload = first_payload_with_keys(inputs, "clipCount", "items")
if payload is None:
raise RuntimeError("成品打包需要上游标注或目标裁剪结果输入")
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
packages = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
clip_name = str(item.get("clipName", "")).strip()
clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem
package_name = f"{clip_id}_package"
package_dir = output_dir / "packages" / package_name
package_dir.mkdir(parents=True, exist_ok=True)
artifact = clip_artifacts.get(clip_name)
clip_path: pathlib.Path | None = None
if artifact is not None:
try:
clip_path = source_path_from_item(
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
clip_name,
output_dir,
archive_sources,
)
except RuntimeError:
clip_path = None
if clip_path is None:
direct_clip_path = str(item.get("clipPath", "")).strip()
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
clip_path = pathlib.Path(direct_clip_path)
if clip_path is None:
clip_path = materialize_clip_from_source(item, output_dir, archive_sources)
if clip_path is not None and clip_path.exists():
shutil.copy2(clip_path, package_dir / clip_name)
quality_payload = {
"clip_id": clip_id,
"source_video": item.get("sourceVideo", ""),
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
}
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
if not annotation_payload and isinstance(item.get("detections"), list):
annotation_payload = {
"type": "target_detections",
"detectionMode": item.get("detectionMode", payload.get("detectionMode", "")),
"targetLabels": item.get("targetLabels", payload.get("targetLabels", [])),
"detections": item.get("detections", []),
"detectionCount": item.get("detectionCount", 0),
"maxConfidence": item.get("maxConfidence", 0),
}
config_payload = package_config_payload(item, payload)
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_quality.json", quality_payload, output=output_binding(node_data))
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation_payload, output=output_binding(node_data))
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_config.json", config_payload, output=output_binding(node_data))
package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()]
packages.append(
{
"packageName": package_name,
"clipId": clip_id,
"clipName": clip_name,
"sourceVideo": item.get("sourceVideo", ""),
"clipDuration": item.get("duration", 0),
"duration": item.get("duration", 0),
"startSecond": item.get("startSecond"),
"endSecond": item.get("endSecond"),
"collectionId": item.get("collectionId", ""),
"meta": item.get("meta", {}),
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
"fileCount": len(package_files),
"pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1"],
"storagePath": str(package_dir),
"quality": quality_payload,
"annotation": annotation_payload,
}
)
manifest = {
"datasetType": "x-platform-packages",
"packageCount": len(packages),
"items": packages,
}
artifacts = [write_json_artifact(run_id, node_id, output_dir, "packages-manifest.json", manifest, output=output_binding(node_data))]
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "packages", node_data=node_data, output=output_binding(node_data)))
return artifacts
def builtin_audit_sampling(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
payload = first_payload_with_keys(inputs, "packageCount", "items")
if payload is None:
raise RuntimeError("抽检采样需要上游成品数据输入")
rules = normalize_sampling_rules(parse_json_param(params, "RULES_JSON", [], list))
sampled_items = []
summaries = []
for rule in rules:
rule_id = rule["id"]
field_path = rule["field"]
operator = rule["operator"]
value = rule.get("value")
percent = rule["samplePercent"]
matched = []
sampled = []
for item in payload.get("items", []):
if not isinstance(item, dict):
continue
if not compare_values(get_field_value(item, field_path), operator, value):
continue
matched.append(item)
package_name = str(item.get("packageName", "")).strip() or str(item.get("clipId", "")).strip()
if stable_sample_hit(f"{rule_id}:{package_name}", percent):
sampled.append(item)
sampled_items.append({**item, "sampleRuleId": rule_id})
summaries.append(
{
"id": rule_id,
"field": field_path,
"operator": operator,
"value": value,
"samplePercent": percent,
"matchedCount": len(matched),
"sampledCount": len(sampled),
}
)
manifest = {
"datasetType": "x-platform-audit-samples",
"rules": summaries,
"sampledCount": len(sampled_items),
"items": sampled_items,
}
return [write_json_artifact(run_id, node_id, output_dir, "audit-samples.json", manifest, output=output_binding(node_data))]
def builtin_condition_router(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
field_path = params.get("FIELD_PATH", "").strip()
operator = normalize_comparison_operator(params.get("OPERATOR", "eq"))
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
rows = collect_items(payloads_from_inputs(inputs))
matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)]
outputs = node_data.get("outputs", []) or [
{"id": "matched", "name": "matched", "type": "json"},
{"id": "unmatched", "name": "unmatched", "type": "json"},
]
return [
write_json_artifact(
run_id,
node_id,
output_dir,
f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json",
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)},
str(outputs[0].get("type", "json")),
output=outputs[0],
),
write_json_artifact(
run_id,
node_id,
output_dir,
f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json",
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)},
str(outputs[1].get("type", "json")),
output=outputs[1],
),
]
def builtin_lerobot_summary(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
if source_path is not None:
root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent)
payload = summarize_lerobot_root(root)
payload["sourcePath"] = str(source_path)
elif inputs:
payload = summarize_lerobot_from_inputs(inputs)
payload["sourcePath"] = ""
else:
raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录")
payload["datasetType"] = "lerobot"
return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))]
def builtin_rosbag_summary(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
params = hyperparams(node_data)
source_path = resolve_source_path(params, inputs)
if source_path is not None:
if source_path.is_dir():
payload = {
"datasetType": "rosbag2",
"sourcePath": str(source_path),
"summary": summarize_directory_tree(source_path),
}
metadata_yaml = source_path / "metadata.yaml"
if metadata_yaml.exists():
payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"]
db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()]
if db3_files:
payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"]
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
payload = summarize_rosbag_file(source_path)
payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag")
payload["sourcePath"] = str(source_path)
else:
for artifact in inputs:
preview = artifact_preview_payload(artifact) or {}
if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}:
payload = {
"datasetType": preview.get("format", "rosbag"),
"sourcePath": str(artifact.get("path", "")),
"preview": preview,
}
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件")
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
def builtin_generic(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_label: str,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
payloads = [load_artifact_payload(item) for item in inputs]
summary = {
"nodeLabel": node_label,
"inputCount": len(inputs),
"inputs": [summarize_json_like(item) for item in payloads],
"config": {
"bucket": node_data.get("bucket", ""),
"prefix": node_data.get("prefix", ""),
"pattern": node_data.get("pattern", ""),
},
}
outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}]
artifacts: list[dict[str, Any]] = []
for index, output in enumerate(outputs):
output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}"
filename = f"{output_name}.json"
payload = {**summary, "outputName": output_name}
artifacts.append(
write_json_artifact(
run_id,
node_id,
output_dir,
filename,
payload,
str(output.get("type", "json")),
output=output,
)
)
return artifacts
def builtin_model_inference(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.model_inference.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_vla_model_train(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.vla_model_train.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_sim3d_asset_generate(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.sim3d_asset_generate.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_holobrain_data_prepare(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.holobrain_data_prepare.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_holobrain_train(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.holobrain_train.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_holobrain_export(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.holobrain_export.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_holobrain_robotwin_eval(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.holobrain_robotwin_eval.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_holobrain_inference_test(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.holobrain_inference_test.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_vla_model_inference(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.vla_model_inference.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_vla_model_evaluate(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.vla_model_evaluate.operator', run_id, node_id, output_dir, node_data, inputs)
def builtin_shell_command(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
return run_operator_module('backend.operators.builtin.shell_command.operator', run_id, node_id, output_dir, node_data, inputs)
def execute_command(
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
command: str,
) -> list[dict[str, Any]]:
validate_command_policy(command)
completed = subprocess.run(
command,
cwd=output_dir,
shell=True,
text=True,
capture_output=True,
env=os.environ.copy(),
check=False,
)
for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)):
for line in content.splitlines():
if line.strip():
log(f"[operator-runner][command][{stream_name}] {line}")
if completed.returncode != 0:
raise RuntimeError(f"command failed with exit code {completed.returncode}")
artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data)
if artifacts:
return artifacts
payload = {"stdout": completed.stdout, "stderr": completed.stderr}
return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))]
OPERATOR_HELPERS = {
"TABULAR_FILE_SUFFIXES": TABULAR_FILE_SUFFIXES,
"DEFAULT_MAX_HAND_OCCLUSION": DEFAULT_MAX_HAND_OCCLUSION,
"DEFAULT_MAX_SHAKE": DEFAULT_MAX_SHAKE,
"DEFAULT_MIN_CLARITY": DEFAULT_MIN_CLARITY,
"RESERVED_MONGO_COLLECTIONS": RESERVED_MONGO_COLLECTIONS,
"VALID_SOURCE_DATA_SOURCES": VALID_SOURCE_DATA_SOURCES,
"VIDEO_FILE_SUFFIXES": VIDEO_FILE_SUFFIXES,
"apply_incremental_source_filter": apply_incremental_source_filter,
"artifact_sha256": artifact_sha256,
"analyze_video_quality_metrics": analyze_video_quality_metrics,
"annotate_video_clip": annotate_video_clip,
"artifact_preview_payload": artifact_preview_payload,
"bos_key_join": bos_key_join,
"bos_prefix": bos_prefix,
"build_bos_client": build_bos_client,
"build_mongo_client": build_mongo_client,
"build_artifact": build_artifact,
"build_virtual_artifact": build_virtual_artifact,
"canonical_bos_path": canonical_bos_path,
"clean_metadata": clean_metadata,
"collect_items": collect_items,
"collect_output_artifacts": collect_output_artifacts,
"common_input_root": common_input_root,
"compare_values": compare_values,
"date_in_range": date_in_range,
"documents_from_inputs": documents_from_inputs,
"execute_command": execute_command,
"extract_preview_data": extract_preview_data,
"find_lerobot_root": find_lerobot_root,
"ffmpeg_available": ffmpeg_available,
"flatten_context": flatten_context,
"first_payload_with_keys": first_payload_with_keys,
"float_value": float_value,
"get_field_value": get_field_value,
"hyperparams": hyperparams,
"incremental_directory_depth": incremental_directory_depth,
"incremental_unit": incremental_unit,
"infer_port_type": infer_port_type,
"inspect_video_file": inspect_video_file,
"json_compatible": json_compatible,
"list_bos_entries": list_bos_entries,
"list_source_entries": list_source_entries,
"load_artifact_payload": load_artifact_payload,
"load_tabular_rows": load_tabular_rows,
"load_payload_from_file": load_payload_from_file,
"load_xplatform_index_payload": load_xplatform_index_payload,
"load_json_file": load_json_file,
"log": log,
"map_json_row": map_json_row,
"materialize_clip_from_source": materialize_clip_from_source,
"materialize_package_artifacts_from_manifest": materialize_package_artifacts_from_manifest,
"materialized_upload_source": materialized_upload_source,
"merge_dicts": merge_dicts,
"normalize_json_payload": normalize_json_payload,
"normalize_date_value": normalize_date_value,
"normalize_comparison_operator": normalize_comparison_operator,
"normalize_field_map": normalize_field_map,
"normalize_sampling_rules": normalize_sampling_rules,
"output_binding": output_binding,
"package_config_payload": package_config_payload,
"package_related_artifact_names": package_related_artifact_names,
"parse_bool_param": parse_bool_param,
"parse_bos_path": parse_bos_path,
"parse_json_param": parse_json_param,
"parse_positive_float_param": parse_positive_float_param,
"parse_positive_int_param": parse_positive_int_param,
"parse_unit_interval_param": parse_unit_interval_param,
"payloads_from_inputs": payloads_from_inputs,
"preview_payload_for_remote_entry": preview_payload_for_remote_entry,
"preview_text_payload": preview_text_payload,
"render_template_text": render_template_text,
"resolve_source_path": resolve_source_path,
"rows_from_payloads": rows_from_payloads,
"source_path_from_item": source_path_from_item,
"source_entry_artifact": source_entry_artifact,
"source_ref_from_artifact": source_ref_from_artifact,
"stable_sample_hit": stable_sample_hit,
"summarize_directory_tree": summarize_directory_tree,
"summarize_input_artifacts": summarize_input_artifacts,
"summarize_lerobot_from_inputs": summarize_lerobot_from_inputs,
"summarize_lerobot_root": summarize_lerobot_root,
"summarize_rosbag_file": summarize_rosbag_file,
"summarize_sqlite_payload": summarize_sqlite_payload,
"text_value": text_value,
"upload_relative_path": upload_relative_path,
"write_csv_artifact": write_csv_artifact,
"write_json_artifact": write_json_artifact,
"write_text_artifact": write_text_artifact,
"write_video_segment": write_video_segment,
"write_video_segment_with_ffmpeg": write_video_segment_with_ffmpeg,
"write_video_segments": write_video_segments,
}
BuiltinHandler = Any
_OPERATOR_MODULE_CACHE: dict[str, Any] = {}
_OPERATOR_MODULE_ALIASES: dict[str, str] = {
f"{path.name}_operator": f"backend.operators.builtin.{path.name}.operator"
for path in (PROJECT_ROOT / "backend" / "operators" / "builtin").iterdir()
if path.is_dir() and (path / "operator.py").exists()
}
def load_operator_module(module_path: str) -> Any:
module = _OPERATOR_MODULE_CACHE.get(module_path)
if module is None:
module = importlib.import_module(module_path)
_OPERATOR_MODULE_CACHE[module_path] = module
return module
def __getattr__(name: str) -> Any:
module_path = _OPERATOR_MODULE_ALIASES.get(name)
if module_path:
return load_operator_module(module_path)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def run_operator_module(
module_path: str,
run_id: str,
node_id: str,
output_dir: pathlib.Path,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
) -> list[dict[str, Any]]:
source_path = os.getenv("DROBOTICFLOW_OPERATOR_SOURCE", "").strip()
if source_path:
source = pathlib.Path(source_path)
if not source.is_file():
raise RuntimeError(f"DROBOTICFLOW_OPERATOR_SOURCE does not exist: {source}")
module_name = "droboticflow_split_operator_" + hashlib.sha1(str(source).encode("utf-8")).hexdigest()
spec = importlib.util.spec_from_file_location(module_name, source)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load operator source: {source}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.run(run_id, node_id, output_dir, node_data, inputs, OPERATOR_HELPERS)
return load_operator_module(module_path).run(run_id, node_id, output_dir, node_data, inputs, OPERATOR_HELPERS)
def operator_handler(module_path: str) -> BuiltinHandler:
def run(run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
return run_operator_module(module_path, run_id, node_id, output_dir, node_data, inputs)
return run
def builtin_generic_handler(run_id: str, node_id: str, output_dir: pathlib.Path, node_label: str, node_data: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
return builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs)
BUILTIN_HANDLER_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("bos_incremental_scan", ("bos 增量扫描", "bos incremental", "增量扫描")),
("human_review", ("人工审核", "人工验收", "任务审核", "human review")),
("mongodb_write", ("mongodb", "mongo 写入", "mongo write", "写入 mongo")),
("xplatform_index", ("x 平台数据集索引", "x-platform", "数据集索引", "dataset index")),
("collection_validate", ("采集目录校验", "目录校验", "validate collection")),
("video_quality", ("视频质检", "quality check", "quality")),
("dataset_filter", ("数据过滤", "dataset filter", "video filter")),
("video_slicing", ("视频切片", "video slicing", "slice")),
("yolo_video_target_clip", ("yolo 视频目标检测裁剪", "yolo target clip", "yolo video detection")),
("video_target_clip", ("视频目标检测裁剪", "目标检测裁剪", "视频目标检测", "target clip", "object detection clip", "video object detection")),
("video_annotation", ("视频标注", "annotation", "annotate")),
("result_package", ("成品打包", "package output", "package result")),
("audit_sampling", ("抽检采样", "audit sample", "sampling")),
("hf_dataset_probe", ("hf 数据集探测", "huggingface dataset", "hf dataset", "dataset viewer")),
("agibot_world2026_validate", ("agibotworld2026", "agibot world", "agibotworld", "智元 world", "智元数据集")),
("bridgedata_tfds_manifest_validate", ("bridgedata", "bridge data", "tfds-rlds", "tfds rlds 清单")),
("rlds_episode_summary", ("rlds", "tfds", "open x-embodiment", "轨迹概览")),
("droid_raw_episode_validate", ("droid raw", "droid episode", "droid trajectory", "droid raw episode", "droid 数据校验")),
("libero_hdf5_task_validate", ("libero hdf5", "libero 任务", "libero 数据集", "libero benchmark")),
("robocasa_hdf5_demonstration_validate", ("robocasa hdf5", "robocasa 示教", "robocasa 数据集", "robocasa demonstration")),
("furniturebench_pkl_demonstration_validate", ("furniturebench", "furniture bench", "家具装配", "furniture pkl")),
("roboturk_real_hdf5_validate", ("roboturk real", "roboturk hdf5", "roboturk aligned", "roboturk 真实", "robo turk")),
("roboset_hdf5_validate", ("roboset hdf5", "roboset 数据", "robo set", "robohive dataset", "robohive hdf5")),
("rh20t_scene_validate", ("rh20t", "rh20t 场景", "rh20t scene", "real human 20t")),
("robomind_hdf5_quality_validate", ("robomind hdf5", "robomind 质量", "robomind 数据", "multi-embodiment intelligence")),
("aloha_hdf5_validate", ("aloha hdf5", "aloha 示教", "act hdf5", "aloha 数据校验")),
("robomimic_hdf5_validate", ("robomimic hdf5", "robomimic 示教", "robomimic 数据校验", "robomimic dataset")),
("hdf5_demonstration_summary", ("hdf5 示教", "hdf5 demo", "robomimic", "droid hdf5")),
("holobrain_data_prepare", ("holobrain 数据准备", "holobrain data prepare", "robotwin2.0 数据准备", "robotwin 数据准备")),
("holobrain_train", ("holobrain 完整训练", "holobrain train", "holobrain 训练")),
("holobrain_export", ("holobrain 模型导出", "holobrain export")),
("holobrain_robotwin_eval", ("holobrain robotwin 评测", "holobrain robotwin eval", "robotwin 评测")),
("holobrain_inference_test", ("holobrain 模型测试", "holobrain inference test", "holobrain 推理测试")),
("openvla_finetune", ("openvla 微调", "openvla finetune", "openvla 训练")),
("openvla_inference", ("openvla 推理", "openvla inference")),
("vla_sample_manifest_validate", ("vla 样本", "vla manifest", "视觉语言动作样本", "droid manifest")),
("rlbench_dataset_validate", ("rlbench", "rlbench 数据集", "rlbench dataset")),
("maniskill_trajectory_validate", ("maniskill", "maniskill 轨迹", "mani skill")),
("calvin_episode_validate", ("calvin episode", "calvin 数据", "calvin episode 校验")),
("rtx_action_schema_validate", ("rt-x 动作", "rtx action", "open x action", "动作空间schema")),
("embodiment_metadata_validate", ("embodiment 元信息", "多机器人embodiment", "robot embodiment", "具身元信息")),
("force_torque_sensor_validate", ("力扭矩", "force torque", "wrench", "ft sensor", "力/扭矩")),
("lerobot_video_alignment_validate", ("lerobot 视频对齐", "lerobot video", "video alignment", "视频对齐校验")),
("lerobot_v3_episode_index_validate", ("lerobot v3 episode", "episode索引", "episode index", "data path template")),
("lerobot_dataset_validate", ("lerobot 数据集校验", "lerobot validate", "lerobot validation")),
("lerobot_metadata_consistency_validate", ("lerobot 元数据", "lerobot metadata", "metadata consistency")),
("trajectory_integrity_validate", ("轨迹多模态完整性校验", "trajectory integrity", "multimodal integrity")),
("action_state_quality_validate", ("动作状态数值", "action state quality", "action/state", "状态数值")),
("episode_boundary_validate", ("episode 边界", "episode boundary", "终止标志", "terminal flag")),
("joint_limit_validate", ("关节限位", "joint limit", "urdf limit", "qpos limit")),
("dataset_split_leakage_validate", ("切分泄漏", "split leakage", "leakage validation", "数据泄漏")),
("dataset_split_manifest", ("训练切分清单", "dataset split", "split manifest")),
("media_reference_validate", ("媒体引用", "media reference", "media readability", "引用可读")),
("dataset_schema_contract_validate", ("schema 合约", "schema contract", "数据集schema", "schema 校验")),
("mcap_topic_summary", ("mcap 主题概览", "mcap topic", "mcap summary")),
("tfrecord_structural_validate", ("tfrecord 结构校验", "tf record", "tfrecord validate")),
("dataset_provenance_audit", ("数据集来源许可审计", "dataset provenance", "license audit", "来源许可")),
("webdataset_shard_summary", ("webdataset 分片", "webdataset shard", "tar shard", "分片概览")),
("diffusion_policy_train", ("diffusion policy 训练", "diffusion policy train", "扩散策略训练")),
("umi_zarr_dataset_validate", ("umi zarr", "umi 数据", "universal manipulation interface", "diffusion policy zarr", "umi dataset")),
("zarr_trajectory_summary", ("zarr 轨迹", "zarr trajectory", "zarr array", "数组轨迹")),
("camera_calibration_validate", ("相机标定校验", "camera calibration", "intrinsics", "extrinsics")),
("tf_transform_tree_validate", ("tf 坐标树", "transform tree", "tf tree", "坐标树校验", "frametree")),
("ros_joint_state_validate", ("ros jointstate", "jointstate", "joint state", "joint_states", "ros 关节状态")),
("graspnet_dataset_validate", ("graspnet", "graspnet-1billion", "抓取数据集", "6d grasp", "general object grasping")),
("bop_dataset_validate", ("bop 6d", "bop 数据集", "bop dataset", "6d姿态", "6d pose", "object pose benchmark")),
("dexycb_dataset_validate", ("dexycb 数据集", "dex-ycb", "dexycb", "ycb hand", "hand grasping object")),
("ho3d_dataset_validate", ("ho3d 数据集", "ho-3d", "ho3d", "h2o3d", "h2o-3d", "honnotate")),
("wilor_hand_pose", ("wilor", "wilor 手部", "手部姿态估计", "3d hand pose", "hand pose estimation")),
("hand_removal", ("hand removal", "视频去手", "手部移除", "去除手臂", "propainter")),
("hand_object_pose_validate", ("手-物体姿态", "手物体姿态", "hand object pose", "dexycb", "ho3d")),
("partnet_mobility_asset_validate", ("partnet-mobility", "partnet mobility", "sapien asset", "sapien 资产", "mobility.urdf", "可交互物体")),
("mjcf_actuator_validate", ("mjcf 执行器", "mjcf actuator", "mujoco actuator", "执行器校验", "ctrlrange")),
("robot_model_asset_validate", ("机器人模型资产校验", "robot model asset", "urdf", "mjcf")),
("pointcloud_summary", ("点云数据概览", "point cloud", "pointcloud", "pcd", "ply")),
("sensor_timestamp_sync_validate", ("多传感器时间同步", "timestamp sync", "sensor sync", "时间同步")),
("parquet_table_summary", ("parquet 表摘要", "parquet summary", "parquet table", "parquet")),
("coco_annotation_validate", ("coco 标注", "coco annotation", "coco validate", "coco")),
("yolo_embodied_detection_train", ("yolo 具身目标检测训练", "yolo detection train", "yolo 训练")),
("yolo_video_target_clip", ("yolo 视频目标检测裁剪", "yolo target clip", "yolo video detection")),
("yolo_detection_validate", ("yolo 检测", "yolo label", "yolo 标注", "yolo")),
("rgbd_frame_pair_validate", ("rgb-d", "rgbd", "rgb-d 帧", "深度帧", "帧对齐")),
("segmentation_mask_validate", ("分割 mask", "segmentation mask", "semantic mask", "mask 校验")),
("language_instruction_validate", ("语言指令", "language instruction", "instruction validate", "vla 指令")),
("lerobot", ("lerobot",)),
("rosbag", ("rosbag", "mcap", "ros bag")),
("csv_to_json", ("csv 转 json", "csv转json", "csv to json", "tsv 转 json", "tsv转json", "tsv to json")),
("json_to_csv", ("json 转 csv", "json转csv", "json to csv")),
("json_extract", ("字段提取", "json提取", "json 提取", "extract")),
("json_filter", ("json过滤", "json 过滤", "filter")),
("json_map", ("json映射", "json 映射", "rename", "映射重命名")),
("text_template", ("模板渲染", "template")),
("condition_router", ("条件分流", "conditional", "router", "route")),
("directory_scan", ("目录扫描", "directory", "browse", "browse dir")),
("merge", ("merge", "合并")),
("intersection", ("intersect", "交集")),
("distribution", ("distribution", "分配")),
("upload", ("upload", "export", "上传", "导出")),
("act_policy_train", ("act 策略训练", "act policy train", "act training")),
("vla_model_train", ("vla 模型训练", "vla training", "视觉语言动作训练", "policy training")),
("sim3d_asset_generate", ("3d 资产仿真生成", "3d资产仿真生成", "仿真资产生成", "simulation asset", "sim3d asset")),
("vla_model_inference", ("vla 模型推理", "vla inference", "视觉语言动作推理", "policy inference")),
("vla_model_evaluate", ("vla 模型评测", "vla 评测", "vla evaluation", "policy evaluation")),
("model_inference", ("模型推理", "model inference", "inference")),
("shell_command", ("shell 命令", "shell command")),
("source", ("source", "scanner", "oss", "scan", "扫描", "数据源")),
)
BUILTIN_HANDLERS: dict[str, BuiltinHandler] = {
"source": operator_handler('backend.operators.builtin.real_source.operator'),
"act_policy_train": operator_handler('backend.operators.builtin.act_policy_train.operator'),
"bos_incremental_scan": operator_handler('backend.operators.builtin.bos_incremental_scan.operator'),
"merge": operator_handler('backend.operators.builtin.merge.operator'),
"intersection": builtin_intersection,
"distribution": builtin_distribution,
"json_extract": operator_handler('backend.operators.builtin.json_field_extract.operator'),
"json_filter": operator_handler('backend.operators.builtin.json_filter.operator'),
"json_map": operator_handler('backend.operators.builtin.json_map_rename.operator'),
"mongodb_write": operator_handler('backend.operators.builtin.mongodb_write.operator'),
"text_template": operator_handler('backend.operators.builtin.template_render.operator'),
"csv_to_json": operator_handler('backend.operators.builtin.csv_to_json.operator'),
"json_to_csv": operator_handler('backend.operators.builtin.json_to_csv.operator'),
"directory_scan": operator_handler('backend.operators.builtin.directory_scan.operator'),
"diffusion_policy_train": operator_handler('backend.operators.builtin.diffusion_policy_train.operator'),
"xplatform_index": operator_handler('backend.operators.builtin.xplatform_dataset_index.operator'),
"collection_validate": operator_handler('backend.operators.builtin.xplatform_dataset_validate.operator'),
"video_quality": operator_handler('backend.operators.builtin.video_quality.operator'),
"dataset_filter": operator_handler('backend.operators.builtin.video_filter.operator'),
"video_slicing": operator_handler('backend.operators.builtin.video_slice.operator'),
"video_target_clip": operator_handler('backend.operators.builtin.video_target_clip.operator'),
"video_annotation": operator_handler('backend.operators.builtin.video_annotate.operator'),
"orbbec_data_normalize": operator_handler('backend.operators.builtin.orbbec_data_normalize.operator'),
"semantic_prelabel": operator_handler('backend.operators.builtin.semantic_prelabel.operator'),
"result_package": operator_handler('backend.operators.builtin.package.operator'),
"audit_sampling": operator_handler('backend.operators.builtin.audit_sample.operator'),
"hf_dataset_probe": operator_handler('backend.operators.builtin.hf_dataset_probe.operator'),
"agibot_world2026_validate": operator_handler('backend.operators.builtin.agibot_world2026_validate.operator'),
"rlds_episode_summary": operator_handler('backend.operators.builtin.rlds_episode_summary.operator'),
"bridgedata_tfds_manifest_validate": operator_handler('backend.operators.builtin.bridgedata_tfds_manifest_validate.operator'),
"droid_raw_episode_validate": operator_handler('backend.operators.builtin.droid_raw_episode_validate.operator'),
"libero_hdf5_task_validate": operator_handler('backend.operators.builtin.libero_hdf5_task_validate.operator'),
"hdf5_demonstration_summary": operator_handler('backend.operators.builtin.hdf5_demonstration_summary.operator'),
"aloha_hdf5_validate": operator_handler('backend.operators.builtin.aloha_hdf5_validate.operator'),
"robomimic_hdf5_validate": operator_handler('backend.operators.builtin.robomimic_hdf5_validate.operator'),
"robocasa_hdf5_demonstration_validate": operator_handler('backend.operators.builtin.robocasa_hdf5_demonstration_validate.operator'),
"furniturebench_pkl_demonstration_validate": operator_handler('backend.operators.builtin.furniturebench_pkl_demonstration_validate.operator'),
"roboturk_real_hdf5_validate": operator_handler('backend.operators.builtin.roboturk_real_hdf5_validate.operator'),
"roboset_hdf5_validate": operator_handler('backend.operators.builtin.roboset_hdf5_validate.operator'),
"rh20t_scene_validate": operator_handler('backend.operators.builtin.rh20t_scene_validate.operator'),
"robomind_hdf5_quality_validate": operator_handler('backend.operators.builtin.robomind_hdf5_quality_validate.operator'),
"vla_sample_manifest_validate": operator_handler('backend.operators.builtin.vla_sample_manifest_validate.operator'),
"rlbench_dataset_validate": operator_handler('backend.operators.builtin.rlbench_dataset_validate.operator'),
"maniskill_trajectory_validate": operator_handler('backend.operators.builtin.maniskill_trajectory_validate.operator'),
"calvin_episode_validate": operator_handler('backend.operators.builtin.calvin_episode_validate.operator'),
"rtx_action_schema_validate": operator_handler('backend.operators.builtin.rtx_action_schema_validate.operator'),
"embodiment_metadata_validate": operator_handler('backend.operators.builtin.embodiment_metadata_validate.operator'),
"force_torque_sensor_validate": operator_handler('backend.operators.builtin.force_torque_sensor_validate.operator'),
"lerobot_video_alignment_validate": operator_handler('backend.operators.builtin.lerobot_video_alignment_validate.operator'),
"lerobot_v3_episode_index_validate": operator_handler('backend.operators.builtin.lerobot_v3_episode_index_validate.operator'),
"lerobot_dataset_validate": operator_handler('backend.operators.builtin.lerobot_dataset_validate.operator'),
"lerobot_metadata_consistency_validate": operator_handler('backend.operators.builtin.lerobot_metadata_consistency_validate.operator'),
"trajectory_integrity_validate": operator_handler('backend.operators.builtin.trajectory_integrity_validate.operator'),
"action_state_quality_validate": operator_handler('backend.operators.builtin.action_state_quality_validate.operator'),
"episode_boundary_validate": operator_handler('backend.operators.builtin.episode_boundary_validate.operator'),
"joint_limit_validate": operator_handler('backend.operators.builtin.joint_limit_validate.operator'),
"dataset_split_leakage_validate": operator_handler('backend.operators.builtin.dataset_split_leakage_validate.operator'),
"dataset_split_manifest": operator_handler('backend.operators.builtin.dataset_split_manifest.operator'),
"media_reference_validate": operator_handler('backend.operators.builtin.media_reference_validate.operator'),
"dataset_schema_contract_validate": operator_handler('backend.operators.builtin.dataset_schema_contract_validate.operator'),
"mcap_topic_summary": operator_handler('backend.operators.builtin.mcap_topic_summary.operator'),
"tfrecord_structural_validate": operator_handler('backend.operators.builtin.tfrecord_structural_validate.operator'),
"dataset_provenance_audit": operator_handler('backend.operators.builtin.dataset_provenance_audit.operator'),
"graspnet_dataset_validate": operator_handler('backend.operators.builtin.graspnet_dataset_validate.operator'),
"bop_dataset_validate": operator_handler('backend.operators.builtin.bop_dataset_validate.operator'),
"dexycb_dataset_validate": operator_handler('backend.operators.builtin.dexycb_dataset_validate.operator'),
"ho3d_dataset_validate": operator_handler('backend.operators.builtin.ho3d_dataset_validate.operator'),
"wilor_hand_pose": operator_handler('backend.operators.builtin.wilor_hand_pose.operator'),
"hand_removal": operator_handler('backend.operators.builtin.hand_removal.operator'),
"webdataset_shard_summary": operator_handler('backend.operators.builtin.webdataset_shard_summary.operator'),
"umi_zarr_dataset_validate": operator_handler('backend.operators.builtin.umi_zarr_dataset_validate.operator'),
"zarr_trajectory_summary": operator_handler('backend.operators.builtin.zarr_trajectory_summary.operator'),
"camera_calibration_validate": operator_handler('backend.operators.builtin.camera_calibration_validate.operator'),
"tf_transform_tree_validate": operator_handler('backend.operators.builtin.tf_transform_tree_validate.operator'),
"ros_joint_state_validate": operator_handler('backend.operators.builtin.ros_joint_state_validate.operator'),
"hand_object_pose_validate": operator_handler('backend.operators.builtin.hand_object_pose_validate.operator'),
"partnet_mobility_asset_validate": operator_handler('backend.operators.builtin.partnet_mobility_asset_validate.operator'),
"mjcf_actuator_validate": operator_handler('backend.operators.builtin.mjcf_actuator_validate.operator'),
"robot_model_asset_validate": operator_handler('backend.operators.builtin.robot_model_asset_validate.operator'),
"pointcloud_summary": operator_handler('backend.operators.builtin.pointcloud_summary.operator'),
"sensor_timestamp_sync_validate": operator_handler('backend.operators.builtin.sensor_timestamp_sync_validate.operator'),
"parquet_table_summary": operator_handler('backend.operators.builtin.parquet_table_summary.operator'),
"coco_annotation_validate": operator_handler('backend.operators.builtin.coco_annotation_validate.operator'),
"yolo_detection_validate": operator_handler('backend.operators.builtin.yolo_detection_validate.operator'),
"rgbd_frame_pair_validate": operator_handler('backend.operators.builtin.rgbd_frame_pair_validate.operator'),
"segmentation_mask_validate": operator_handler('backend.operators.builtin.segmentation_mask_validate.operator'),
"language_instruction_validate": operator_handler('backend.operators.builtin.language_instruction_validate.operator'),
"condition_router": operator_handler('backend.operators.builtin.conditional_route.operator'),
"human_review": operator_handler('backend.operators.builtin.human_review.operator'),
"builtin_human_review": operator_handler('backend.operators.builtin.human_review.operator'),
"lerobot": operator_handler('backend.operators.builtin.lerobot_summary.operator'),
"rosbag": operator_handler('backend.operators.builtin.rosbag_summary.operator'),
"upload": operator_handler('backend.operators.builtin.upload.operator'),
"holobrain_data_prepare": operator_handler('backend.operators.builtin.holobrain_data_prepare.operator'),
"holobrain_train": operator_handler('backend.operators.builtin.holobrain_train.operator'),
"holobrain_export": operator_handler('backend.operators.builtin.holobrain_export.operator'),
"holobrain_robotwin_eval": operator_handler('backend.operators.builtin.holobrain_robotwin_eval.operator'),
"holobrain_inference_test": operator_handler('backend.operators.builtin.holobrain_inference_test.operator'),
"openvla_finetune": operator_handler('backend.operators.builtin.openvla_finetune.operator'),
"openvla_inference": operator_handler('backend.operators.builtin.openvla_inference.operator'),
"vla_model_train": operator_handler('backend.operators.builtin.vla_model_train.operator'),
"sim3d_asset_generate": operator_handler('backend.operators.builtin.sim3d_asset_generate.operator'),
"vla_model_inference": operator_handler('backend.operators.builtin.vla_model_inference.operator'),
"vla_model_evaluate": operator_handler('backend.operators.builtin.vla_model_evaluate.operator'),
"model_inference": builtin_model_inference,
"yolo_embodied_detection_train": operator_handler('backend.operators.builtin.yolo_embodied_detection_train.operator'),
"yolo_video_target_clip": operator_handler('backend.operators.builtin.yolo_video_target_clip.operator'),
"shell_command": builtin_shell_command,
"generic": builtin_generic_handler,
}
for _handler_name, _handler in tuple(BUILTIN_HANDLERS.items()):
if _handler_name != "generic":
BUILTIN_HANDLERS.setdefault(f"builtin_{_handler_name}", _handler)
BUILTIN_HANDLERS.setdefault("builtin_lerobot_summary", BUILTIN_HANDLERS["lerobot"])
BUILTIN_HANDLERS.setdefault("builtin_rosbag_summary", BUILTIN_HANDLERS["rosbag"])
def select_handler(node_label: str) -> str:
normalized = node_label.lower()
for handler, tokens in BUILTIN_HANDLER_RULES:
if any(token in normalized for token in tokens):
return handler
return "generic"
def runtime_handler_override() -> str:
handler = os.getenv("DROBOTICFLOW_OPERATOR_HANDLER", "").strip()
if not handler:
return ""
if handler not in BUILTIN_HANDLERS:
known = ", ".join(sorted(BUILTIN_HANDLERS))
raise RuntimeError(f"DROBOTICFLOW_OPERATOR_HANDLER={handler!r} is not registered; known handlers: {known}")
return handler
def main() -> int:
run_id = os.getenv("RUN_ID", "").strip()
workflow_id = os.getenv("WORKFLOW_ID", "").strip()
node_id = os.getenv("NODE_ID", "").strip()
node_label = os.getenv("NODE_LABEL", "").strip() or node_id
output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id)))
output_dir.mkdir(parents=True, exist_ok=True)
node_data = load_json_env("NODE_DATA_JSON", {})
inputs = load_input_artifacts()
started_at = now_iso()
start_cpu, start_memory = resource_snapshot()
sampler = ResourceSampler(start_cpu, start_memory, started_at)
sampler.start()
logs: list[dict[str, Any]] = []
def append_log(level: str, message: str) -> None:
logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label})
log(f"[operator-runner][{level.lower()}] {message}")
append_log("INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir}")
append_log("INFO", "Preparing operator execution context")
try:
params = hyperparams(node_data)
command = params.get("COMMAND") or params.get("SCRIPT")
artifacts: list[dict[str, Any]]
if command:
append_log("INFO", "Executing explicit command")
artifacts = execute_command(run_id, node_id, output_dir, node_data, command)
else:
handler = runtime_handler_override() or select_handler(node_label)
append_log("INFO", f"Selected built-in handler: {handler}")
builtin_handler = BUILTIN_HANDLERS.get(handler, BUILTIN_HANDLERS["generic"])
if handler == "generic":
artifacts = builtin_handler(run_id, node_id, output_dir, node_label, node_data, inputs)
else:
artifacts = builtin_handler(run_id, node_id, output_dir, node_data, inputs)
finally:
ended_at = now_iso()
resource_samples = sampler.stop(ended_at)
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
data_metrics = infer_data_metrics(node_data, artifacts, inputs)
result = {
"logs": logs,
"artifacts": artifacts,
"resourceSeries": resource_samples,
}
if data_metrics:
result["dataMetrics"] = data_metrics
stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip()
if stage_manifest_path:
write_json_file(
stage_manifest_path,
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
)
else:
write_json_file(
default_stage_manifest_path(output_dir),
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
)
result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip()
if result_output_path:
written_path = write_json_file(result_output_path, result)
print(RESULT_PATH_PREFIX + str(written_path), flush=True)
return 0
default_result_path = write_json_file(default_result_output_path(output_dir), result)
print(RESULT_PATH_PREFIX + str(default_result_path), flush=True)
return 0
def tempfile_dir(run_id: str, node_id: str) -> str:
base = pathlib.Path("/tmp/droboticflow")
return str(base / (run_id or "run") / (node_id or "node"))
if __name__ == "__main__":
try:
sys.exit(main())
except Exception as error: # pragma: no cover
log(f"[operator-runner][error] {error}")
sys.exit(1)