forked from longtao.wu/droboticflow
5373 lines
221 KiB
Python
5373 lines
221 KiB
Python
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import datetime as dt
|
||
import contextlib
|
||
import fnmatch
|
||
import hashlib
|
||
import importlib
|
||
import json
|
||
import mimetypes
|
||
import os
|
||
import pathlib
|
||
import base64
|
||
import csv
|
||
import math
|
||
import re
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import tarfile
|
||
import time
|
||
import zipfile
|
||
from typing import Any
|
||
from urllib import error, request
|
||
from urllib.parse import urlparse
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||
if str(PROJECT_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(PROJECT_ROOT))
|
||
RUNNER_ROOT = pathlib.Path(__file__).resolve().parent
|
||
if str(RUNNER_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(RUNNER_ROOT))
|
||
|
||
from resource_monitor import ResourceSampler, resource_snapshot
|
||
|
||
RESULT_PREFIX = "DROBOTICFLOW_RESULT="
|
||
RESULT_PATH_PREFIX = "DROBOTICFLOW_RESULT_PATH="
|
||
TEXT_FILE_SUFFIXES = {
|
||
".txt",
|
||
".log",
|
||
".md",
|
||
".yaml",
|
||
".yml",
|
||
".xml",
|
||
".html",
|
||
".htm",
|
||
}
|
||
TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"}
|
||
IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
|
||
VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
|
||
AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"}
|
||
ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"}
|
||
ARCHIVE_FILE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip")
|
||
XPLATFORM_VIDEO_META_FILES = ("video_meta.json", "video_metadata.json")
|
||
XPLATFORM_INLINE_JSON_FILES = {"meta.json", "intrinsics.json", *XPLATFORM_VIDEO_META_FILES}
|
||
PREVIEW_TEXT_LIMIT = 12_000
|
||
PREVIEW_TEXT_LINE_LIMIT = 240
|
||
PREVIEW_TABLE_ROW_LIMIT = 24
|
||
PREVIEW_FILE_LIST_LIMIT = 40
|
||
PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024
|
||
DATASET_PARTITION_SIZE = 500
|
||
QUALITY_SAMPLE_LIMIT = 48
|
||
QUALITY_ANALYSIS_MAX_DIMENSION = 640
|
||
QUALITY_SKIN_COMPONENT_MIN_RATIO = 0.002
|
||
DEFAULT_MIN_CLARITY = 0.8
|
||
DEFAULT_MAX_SHAKE = 0.1
|
||
DEFAULT_MAX_HAND_OCCLUSION = 0.1
|
||
TRUE_ENV_VALUES = {"1", "true", "yes", "on"}
|
||
FALSE_ENV_VALUES = {"0", "false", "no", "off"}
|
||
MAX_SAFE_INTEGER = 9_007_199_254_740_991
|
||
EXACT_POSITIVE_INT_RE = re.compile(r"^[1-9][0-9]*$")
|
||
INTEGERISH_RE = re.compile(r"^[+-]?[0-9]+$")
|
||
SUPPORTED_COMPARISON_OPERATORS = {"contains", "eq", "exists", "gt", "gte", "in", "lt", "lte", "missing", "ne"}
|
||
VALID_SOURCE_DATA_SOURCES = {"INLINE", "LOCAL", "BOS", "UPSTREAM"}
|
||
RESERVED_MONGO_COLLECTIONS = {
|
||
"projects",
|
||
"operators",
|
||
"workflows",
|
||
"templates",
|
||
"runs",
|
||
"sourceStates",
|
||
"auditTasks",
|
||
"samplingTasks",
|
||
"users",
|
||
"storageConnections",
|
||
"settings",
|
||
}
|
||
|
||
|
||
def env_bool(name: str, default: bool) -> bool:
|
||
value = os.getenv(name, "").strip().lower()
|
||
if value == "":
|
||
return default
|
||
if value in TRUE_ENV_VALUES:
|
||
return True
|
||
if value in FALSE_ENV_VALUES:
|
||
return False
|
||
raise RuntimeError(f"{name} must be one of true/false/1/0/yes/no/on/off")
|
||
|
||
|
||
def compile_policy_regex(name: str) -> re.Pattern[str] | None:
|
||
value = os.getenv(name, "").strip()
|
||
if not value:
|
||
return None
|
||
try:
|
||
return re.compile(value)
|
||
except re.error as error:
|
||
raise RuntimeError(f"{name} is not a valid regular expression: {error}") from error
|
||
|
||
|
||
def validate_command_policy(command: str) -> None:
|
||
if not env_bool("OPERATOR_ALLOW_SHELL_COMMANDS", False):
|
||
raise RuntimeError("explicit shell commands are disabled by OPERATOR_ALLOW_SHELL_COMMANDS")
|
||
|
||
denylist = compile_policy_regex("OPERATOR_COMMAND_DENYLIST_REGEX")
|
||
if denylist and denylist.search(command):
|
||
raise RuntimeError("explicit shell command was rejected by OPERATOR_COMMAND_DENYLIST_REGEX")
|
||
|
||
allowlist = compile_policy_regex("OPERATOR_COMMAND_ALLOWLIST_REGEX")
|
||
if allowlist and not allowlist.search(command):
|
||
raise RuntimeError("explicit shell command was not allowed by OPERATOR_COMMAND_ALLOWLIST_REGEX")
|
||
DATASET_PARTITION_KEYS = (
|
||
"items",
|
||
"entries",
|
||
"rows",
|
||
"collections",
|
||
"catalogCollections",
|
||
"catalogItems",
|
||
"catalogEntries",
|
||
)
|
||
PAYLOAD_FILE_READ_ATTEMPTS = 60
|
||
PAYLOAD_FILE_READ_DELAY_SECONDS = 1.0
|
||
|
||
|
||
def now_iso() -> str:
|
||
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||
|
||
|
||
def log(message: str) -> None:
|
||
print(message, flush=True)
|
||
|
||
|
||
def load_json_env(name: str, default: Any) -> Any:
|
||
value = os.getenv(name, "").strip()
|
||
if not value:
|
||
return default
|
||
return json.loads(value)
|
||
|
||
|
||
def load_json_file(path: str, default: Any) -> Any:
|
||
raw = str(path or "").strip()
|
||
if not raw:
|
||
return default
|
||
candidate = pathlib.Path(raw)
|
||
if not candidate.exists():
|
||
return default
|
||
return json.loads(candidate.read_text(encoding="utf-8"))
|
||
|
||
|
||
def write_json_file(path: str | pathlib.Path, value: Any) -> pathlib.Path:
|
||
candidate = pathlib.Path(path)
|
||
candidate.parent.mkdir(parents=True, exist_ok=True)
|
||
candidate.write_text(json.dumps(json_compatible(value), ensure_ascii=False), encoding="utf-8")
|
||
return candidate
|
||
|
||
|
||
def text_value(value: Any) -> str:
|
||
if isinstance(value, bytes):
|
||
return value.decode("utf-8", errors="replace")
|
||
return str(value or "")
|
||
|
||
|
||
def json_compatible(value: Any) -> Any:
|
||
if isinstance(value, bytes):
|
||
return value.decode("utf-8", errors="replace")
|
||
if isinstance(value, dict):
|
||
return {str(key): json_compatible(item) for key, item in value.items()}
|
||
if isinstance(value, list):
|
||
return [json_compatible(item) for item in value]
|
||
if isinstance(value, tuple):
|
||
return [json_compatible(item) for item in value]
|
||
return value
|
||
|
||
|
||
def load_input_artifacts() -> list[dict[str, Any]]:
|
||
input_artifacts_path = os.getenv("INPUT_ARTIFACTS_PATH", "").strip()
|
||
if input_artifacts_path:
|
||
loaded = load_json_file(input_artifacts_path, [])
|
||
if isinstance(loaded, list):
|
||
return json_compatible(loaded)
|
||
loaded = load_json_env("INPUT_ARTIFACTS_JSON", [])
|
||
if isinstance(loaded, list):
|
||
return json_compatible(loaded)
|
||
return []
|
||
|
||
|
||
def load_input_bindings() -> list[dict[str, Any]]:
|
||
input_bindings_path = os.getenv("INPUT_BINDINGS_PATH", "").strip()
|
||
if input_bindings_path:
|
||
loaded = load_json_file(input_bindings_path, [])
|
||
if isinstance(loaded, list):
|
||
return json_compatible(loaded)
|
||
loaded = load_json_env("INPUT_BINDINGS_JSON", [])
|
||
if isinstance(loaded, list):
|
||
return json_compatible(loaded)
|
||
return []
|
||
|
||
|
||
def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None:
|
||
for key in keys:
|
||
value = payload.get(key)
|
||
if isinstance(value, bool):
|
||
continue
|
||
if isinstance(value, (int, float)) and value >= 0:
|
||
return value
|
||
return None
|
||
|
||
|
||
def _metric_volume(payloads: list[dict[str, Any]]) -> dict[str, int | float]:
|
||
candidates: dict[str, list[int | float]] = {
|
||
"trajectoryCount": [],
|
||
"durationSeconds": [],
|
||
"bytes": [],
|
||
"fileCount": [],
|
||
}
|
||
for payload in payloads:
|
||
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
|
||
trajectory_count = _first_numeric(
|
||
payload,
|
||
("trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"),
|
||
)
|
||
if trajectory_count is None and data_ref:
|
||
trajectory_count = _first_numeric(data_ref, ("itemCount",))
|
||
duration_seconds = _first_numeric(payload, ("durationSeconds", "totalDurationSeconds", "totalDuration", "duration"))
|
||
total_bytes = _first_numeric(payload, ("totalBytes", "bytes", "sizeBytes"))
|
||
file_count = _first_numeric(payload, ("fileCount", "filesCount"))
|
||
if trajectory_count is not None:
|
||
candidates["trajectoryCount"].append(int(trajectory_count))
|
||
if duration_seconds is not None:
|
||
candidates["durationSeconds"].append(float(duration_seconds))
|
||
if total_bytes is not None:
|
||
candidates["bytes"].append(int(total_bytes))
|
||
if file_count is not None:
|
||
candidates["fileCount"].append(int(file_count))
|
||
return {key: max(values) for key, values in candidates.items() if values}
|
||
|
||
|
||
def _artifact_payloads(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
return [item["data"] for item in artifacts if isinstance(item.get("data"), dict)]
|
||
|
||
|
||
def _dataset_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
refs: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for artifact in artifacts:
|
||
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
|
||
if not data_ref:
|
||
continue
|
||
item = {
|
||
"name": text_value(artifact.get("name")),
|
||
"kind": text_value(data_ref.get("kind")),
|
||
"datasetType": text_value(data_ref.get("datasetType")),
|
||
"manifestPath": text_value(data_ref.get("manifestPath")),
|
||
"payloadPath": text_value(data_ref.get("payloadPath")),
|
||
"partitionDir": text_value(data_ref.get("partitionDir")),
|
||
"partitionCount": int(data_ref.get("partitionCount") or 0),
|
||
"itemCount": int(data_ref.get("itemCount") or 0),
|
||
"datasetId": text_value(data_ref.get("datasetId")),
|
||
"datasetVersionId": text_value(data_ref.get("datasetVersionId")),
|
||
"split": text_value(data_ref.get("split")),
|
||
"fingerprint": text_value(data_ref.get("fingerprint")),
|
||
}
|
||
key = "|".join(
|
||
text_value(item.get(name))
|
||
for name in ("datasetVersionId", "datasetId", "manifestPath", "payloadPath", "partitionDir", "split")
|
||
)
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
refs.append(item)
|
||
return refs
|
||
|
||
|
||
def infer_data_metrics(
|
||
node_data: dict[str, Any],
|
||
artifacts: list[dict[str, Any]],
|
||
inputs: list[dict[str, Any]],
|
||
) -> dict[str, Any] | None:
|
||
stage = text_value(node_data.get("businessStage")).strip().lower()
|
||
if stage not in {"collect", "process", "clean", "validate", "annotate", "audit", "train"}:
|
||
return None
|
||
if node_data.get("metricBoundary") is False:
|
||
return None
|
||
output_payloads = _artifact_payloads(artifacts)
|
||
for payload in output_payloads:
|
||
explicit = payload.get("dataMetrics")
|
||
if isinstance(explicit, dict):
|
||
normalized = json_compatible(explicit)
|
||
normalized.setdefault("schemaVersion", 1)
|
||
normalized.setdefault("stage", stage)
|
||
normalized.setdefault("quality", "exact")
|
||
normalized.setdefault("occurredAt", now_iso())
|
||
return normalized
|
||
|
||
input_payloads = _artifact_payloads(inputs)
|
||
input_volume = _metric_volume(input_payloads)
|
||
output_volume = _metric_volume(output_payloads)
|
||
metrics: dict[str, Any] = {
|
||
"schemaVersion": 1,
|
||
"stage": stage,
|
||
"input": input_volume,
|
||
"output": output_volume,
|
||
"datasetRefs": _dataset_refs(artifacts),
|
||
"quality": "derived",
|
||
"occurredAt": now_iso(),
|
||
}
|
||
if stage == "collect" and not input_volume:
|
||
metrics.pop("input")
|
||
if stage == "validate":
|
||
valid = any(payload.get("valid") is True for payload in output_payloads)
|
||
metrics["technicalValid"] = output_volume if valid else {}
|
||
if not valid and input_volume:
|
||
metrics["rejected"] = input_volume
|
||
if stage in {"annotate", "audit"}:
|
||
metrics["trainReady"] = output_volume
|
||
if stage == "clean" and input_volume and output_volume:
|
||
rejected: dict[str, int | float] = {}
|
||
for key in ("trajectoryCount", "durationSeconds", "bytes", "fileCount"):
|
||
if key in input_volume and key in output_volume:
|
||
rejected[key] = max(0, input_volume[key] - output_volume[key])
|
||
if rejected:
|
||
metrics["rejected"] = rejected
|
||
if stage == "train":
|
||
training_inputs: list[dict[str, Any]] = []
|
||
for ref in _dataset_refs(inputs):
|
||
training_inputs.append(
|
||
{
|
||
"datasetId": ref.get("datasetId", ""),
|
||
"datasetVersionId": ref.get("datasetVersionId", ""),
|
||
"name": ref.get("name", ""),
|
||
"split": ref.get("split", ""),
|
||
"volume": input_volume,
|
||
"manifestPath": ref.get("manifestPath", ""),
|
||
"partitionDir": ref.get("partitionDir", ""),
|
||
"external": not bool(ref.get("datasetVersionId")),
|
||
"quality": "derived",
|
||
}
|
||
)
|
||
metrics["trainingInputs"] = training_inputs
|
||
return metrics
|
||
|
||
|
||
def stage_manifest_payload(
|
||
run_id: str,
|
||
workflow_id: str,
|
||
node_id: str,
|
||
node_label: str,
|
||
output_dir: pathlib.Path,
|
||
artifacts: list[dict[str, Any]],
|
||
data_metrics: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
manifest_artifacts: list[dict[str, Any]] = []
|
||
data_refs: list[dict[str, Any]] = []
|
||
for artifact in artifacts:
|
||
item = {
|
||
"id": text_value(artifact.get("id")),
|
||
"name": text_value(artifact.get("name")),
|
||
"type": text_value(artifact.get("type")),
|
||
"path": text_value(artifact.get("path")),
|
||
"size": text_value(artifact.get("size")),
|
||
}
|
||
if text_value(artifact.get("portId")):
|
||
item["portId"] = text_value(artifact.get("portId"))
|
||
if text_value(artifact.get("portName")):
|
||
item["portName"] = text_value(artifact.get("portName"))
|
||
if isinstance(artifact.get("metadata"), dict) and artifact["metadata"]:
|
||
item["metadata"] = json_compatible(artifact["metadata"])
|
||
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
|
||
if data_ref:
|
||
item["dataRef"] = json_compatible(data_ref)
|
||
data_refs.append(
|
||
{
|
||
"artifactId": item["id"],
|
||
"artifactName": item["name"],
|
||
"portId": item.get("portId", ""),
|
||
**json_compatible(data_ref),
|
||
}
|
||
)
|
||
manifest_artifacts.append(item)
|
||
result = {
|
||
"runId": run_id,
|
||
"workflowId": workflow_id,
|
||
"nodeId": node_id,
|
||
"nodeLabel": node_label,
|
||
"outputDir": str(output_dir),
|
||
"artifactCount": len(manifest_artifacts),
|
||
"artifacts": manifest_artifacts,
|
||
"dataRefs": data_refs,
|
||
"primaryDataRefs": data_refs,
|
||
"generatedAt": now_iso(),
|
||
}
|
||
if data_metrics:
|
||
result["dataMetrics"] = json_compatible(data_metrics)
|
||
return result
|
||
|
||
|
||
def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path:
|
||
return output_dir.parent / "result.json"
|
||
|
||
|
||
def default_stage_manifest_path(output_dir: pathlib.Path) -> pathlib.Path:
|
||
return output_dir.parent / "stage-manifest.json"
|
||
|
||
|
||
def normalize_json_payload(value: Any) -> dict[str, Any]:
|
||
if isinstance(value, dict):
|
||
return json_compatible(value)
|
||
if isinstance(value, list):
|
||
return {"items": json_compatible(value), "itemCount": len(value)}
|
||
return {"value": json_compatible(value)}
|
||
|
||
|
||
def bos_prefix(value: str) -> str:
|
||
return str(value or "").strip().strip("/")
|
||
|
||
|
||
def canonical_bos_path(bucket: str, key: str) -> str:
|
||
normalized_bucket = str(bucket or "").strip()
|
||
normalized_key = str(key or "").strip().lstrip("/")
|
||
if not normalized_bucket:
|
||
return ""
|
||
if not normalized_key:
|
||
return f"bos://{normalized_bucket}"
|
||
return f"bos://{normalized_bucket}/{normalized_key}"
|
||
|
||
|
||
def parse_bos_path(value: str) -> tuple[str, str]:
|
||
raw = str(value or "").strip()
|
||
if not raw.startswith("bos://"):
|
||
return "", ""
|
||
bucket_and_key = raw[len("bos://") :]
|
||
if not bucket_and_key:
|
||
return "", ""
|
||
bucket, _, key = bucket_and_key.partition("/")
|
||
return bucket.strip(), bos_prefix(key)
|
||
|
||
|
||
def is_xplatform_video_meta_name(value: str) -> bool:
|
||
return str(value or "").strip().lower() in XPLATFORM_VIDEO_META_FILES
|
||
|
||
|
||
def load_collection_video_meta(collection_dir: pathlib.Path) -> tuple[dict[str, dict[str, Any]], str]:
|
||
for candidate in XPLATFORM_VIDEO_META_FILES:
|
||
payload = optional_json_file(collection_dir / candidate)
|
||
if payload is not None:
|
||
return parse_video_meta_rows(payload), candidate
|
||
return {}, ""
|
||
|
||
|
||
def sampled_frame_positions(frame_count: int, sample_limit: int = QUALITY_SAMPLE_LIMIT) -> list[int]:
|
||
if frame_count <= 0:
|
||
return []
|
||
if frame_count <= sample_limit:
|
||
return list(range(frame_count))
|
||
last_frame = max(frame_count - 1, 0)
|
||
positions = {
|
||
min(last_frame, max(0, int(round(index * last_frame / max(sample_limit - 1, 1)))))
|
||
for index in range(sample_limit)
|
||
}
|
||
return sorted(positions)
|
||
|
||
|
||
def resize_quality_frame(frame: Any, max_dimension: int = QUALITY_ANALYSIS_MAX_DIMENSION) -> Any:
|
||
try:
|
||
height, width = frame.shape[:2]
|
||
except Exception:
|
||
return frame
|
||
longest_edge = max(width, height)
|
||
if longest_edge <= 0 or longest_edge <= max_dimension:
|
||
return frame
|
||
scale = max_dimension / float(longest_edge)
|
||
target_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
|
||
cv2, _ = require_cv2()
|
||
return cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA)
|
||
|
||
|
||
def build_skin_mask(frame: Any) -> Any:
|
||
cv2, np = require_cv2()
|
||
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
|
||
ycrcb_mask = cv2.inRange(
|
||
ycrcb,
|
||
np.array((0, 133, 77), dtype=np.uint8),
|
||
np.array((255, 173, 127), dtype=np.uint8),
|
||
)
|
||
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
|
||
hsv_mask = cv2.inRange(
|
||
hsv,
|
||
np.array((0, 20, 40), dtype=np.uint8),
|
||
np.array((25, 220, 255), dtype=np.uint8),
|
||
)
|
||
return cv2.bitwise_and(ycrcb_mask, hsv_mask)
|
||
|
||
|
||
def largest_skin_component_ratio(mask: Any, min_component_ratio: float = QUALITY_SKIN_COMPONENT_MIN_RATIO) -> float:
|
||
cv2, np = require_cv2()
|
||
if mask is None:
|
||
return 0.0
|
||
total_area = int(mask.shape[0] * mask.shape[1]) if len(mask.shape) >= 2 else 0
|
||
if total_area <= 0:
|
||
return 0.0
|
||
kernel = np.ones((5, 5), dtype=np.uint8)
|
||
cleaned = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
|
||
cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_CLOSE, kernel)
|
||
component_threshold = max(1, int(total_area * max(min_component_ratio, 0.0)))
|
||
count, _, stats, _ = cv2.connectedComponentsWithStats(cleaned)
|
||
largest_area = 0
|
||
for index in range(1, count):
|
||
area = int(stats[index, cv2.CC_STAT_AREA])
|
||
if area < component_threshold:
|
||
continue
|
||
if area > largest_area:
|
||
largest_area = area
|
||
return float(largest_area / total_area)
|
||
|
||
|
||
def motion_jitter_ratio(shifts: list[tuple[float, float]], diagonal: float) -> float:
|
||
if not shifts:
|
||
return 0.0
|
||
_, np = require_cv2()
|
||
normalized_diagonal = max(diagonal, 1.0)
|
||
dx_values = [item[0] for item in shifts]
|
||
dy_values = [item[1] for item in shifts]
|
||
median_dx = float(np.median(dx_values))
|
||
median_dy = float(np.median(dy_values))
|
||
jitters = [
|
||
math.sqrt((dx - median_dx) ** 2 + (dy - median_dy) ** 2) / normalized_diagonal
|
||
for dx, dy in shifts
|
||
]
|
||
return sum(jitters) / max(len(jitters), 1)
|
||
|
||
|
||
def bos_key_join(prefix: str, relative_path: str) -> str:
|
||
normalized_prefix = bos_prefix(prefix)
|
||
normalized_relative = str(relative_path or "").strip().lstrip("/")
|
||
if not normalized_prefix:
|
||
return normalized_relative
|
||
if not normalized_relative:
|
||
return normalized_prefix
|
||
return f"{normalized_prefix}/{normalized_relative}"
|
||
|
||
|
||
def require_bos_sdk():
|
||
try:
|
||
import baidubce.protocol as protocol # type: ignore
|
||
from baidubce.auth.bce_credentials import BceCredentials # type: ignore
|
||
from baidubce.bce_client_configuration import BceClientConfiguration # type: ignore
|
||
from baidubce.services.bos.bos_client import BosClient # type: ignore
|
||
except Exception as error: # pragma: no cover - runtime environment specific
|
||
raise RuntimeError("BOS 扫描需要 bce-python-sdk 运行时") from error
|
||
return protocol, BceCredentials, BceClientConfiguration, BosClient
|
||
|
||
|
||
def build_bos_client(endpoint: str, access_key_id: str, secret_access_key: str):
|
||
protocol_module, BceCredentials, BceClientConfiguration, BosClient = require_bos_sdk()
|
||
parsed = urlparse(endpoint if "://" in endpoint else f"https://{endpoint}")
|
||
scheme = (parsed.scheme or "https").lower()
|
||
host = parsed.netloc or parsed.path
|
||
if not host:
|
||
raise RuntimeError("BOS endpoint 不能为空")
|
||
config = BceClientConfiguration(
|
||
credentials=BceCredentials(access_key_id, secret_access_key),
|
||
endpoint=host,
|
||
protocol=protocol_module.HTTPS if scheme == "https" else protocol_module.HTTP,
|
||
)
|
||
return BosClient(config), scheme
|
||
|
||
|
||
def format_size(num_bytes: int) -> str:
|
||
if num_bytes < 1024:
|
||
return f"{num_bytes} B"
|
||
if num_bytes < 1024 * 1024:
|
||
return f"{num_bytes / 1024:.1f} KiB"
|
||
if num_bytes < 1024 * 1024 * 1024:
|
||
return f"{num_bytes / (1024 * 1024):.1f} MiB"
|
||
return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB"
|
||
|
||
|
||
MAX_SHA256_FILE_BYTES = 64 * 1024 * 1024
|
||
|
||
|
||
def sha256_file(path: pathlib.Path) -> str:
|
||
digest = hashlib.sha256()
|
||
with path.open("rb") as handle:
|
||
for chunk in iter(lambda: handle.read(65536), b""):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def artifact_sha256(path: pathlib.Path, size_bytes: int) -> tuple[str, dict[str, Any]]:
|
||
if size_bytes > MAX_SHA256_FILE_BYTES:
|
||
return "", {"sha256Skipped": True, "sha256SkipReason": "size-limit", "sha256SkipBytes": size_bytes}
|
||
return sha256_file(path), {}
|
||
|
||
|
||
def require_cv2() -> tuple[Any, Any]:
|
||
try:
|
||
import cv2 # type: ignore
|
||
import numpy as np # type: ignore
|
||
except Exception as error: # pragma: no cover - environment specific
|
||
raise RuntimeError("视频算子需要 opencv-python-headless 和 numpy 运行时") from error
|
||
return cv2, np
|
||
|
||
|
||
def data_url_for_path(path: pathlib.Path, mime_type: str) -> str:
|
||
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
||
return f"data:{mime_type};base64,{encoded}"
|
||
|
||
|
||
def decode_data_url(url: str) -> tuple[str, bytes] | None:
|
||
if not url.startswith("data:") or "," not in url:
|
||
return None
|
||
header, encoded = url.split(",", 1)
|
||
if ";base64" not in header:
|
||
return None
|
||
mime_type = header[5:].split(";", 1)[0] or "application/octet-stream"
|
||
try:
|
||
return mime_type, base64.b64decode(encoded)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def download_remote_source_to_path(url: str, path: pathlib.Path) -> pathlib.Path | None:
|
||
parsed = urlparse(url)
|
||
if parsed.scheme.lower() not in {"http", "https"}:
|
||
return None
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
remote_request = request.Request(
|
||
url,
|
||
headers={
|
||
"User-Agent": "DroboticFlow operator-runner/1.0",
|
||
},
|
||
)
|
||
try:
|
||
with request.urlopen(remote_request, timeout=300) as response, path.open("wb") as handle:
|
||
shutil.copyfileobj(response, handle, length=1024 * 1024)
|
||
except error.HTTPError as exc:
|
||
raise RuntimeError(f"下载远程输入失败({exc.code}): {path.name}") from exc
|
||
except error.URLError as exc:
|
||
raise RuntimeError(f"下载远程输入失败: {path.name}") from exc
|
||
return path
|
||
|
||
|
||
def download_remote_source(url: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
|
||
path = output_dir / "_materialized_sources" / default_name
|
||
return download_remote_source_to_path(url, path)
|
||
|
||
|
||
def workspace_cached_bos_source_path(raw_path: str, output_dir: pathlib.Path) -> pathlib.Path | None:
|
||
bucket, key = parse_bos_path(raw_path)
|
||
if not bucket or not key:
|
||
return None
|
||
workspace_root = str(os.getenv("WORKSPACE_ROOT", "")).strip()
|
||
if workspace_root:
|
||
root = pathlib.Path(workspace_root)
|
||
if root.is_absolute():
|
||
return root / "_bos_cache" / bucket / key
|
||
return output_dir / "_materialized_sources" / bucket / key
|
||
|
||
|
||
def download_bos_source(raw_path: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
|
||
target = workspace_cached_bos_source_path(raw_path, output_dir)
|
||
if target is None:
|
||
return None
|
||
if target.exists() and target.stat().st_size > 0:
|
||
return target
|
||
bucket, key = parse_bos_path(raw_path)
|
||
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
if not bucket or not key or not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
|
||
return None
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
temp_target = target.with_name(target.name + ".part")
|
||
with contextlib.suppress(FileNotFoundError):
|
||
temp_target.unlink()
|
||
try:
|
||
client, _ = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
||
client.get_object_to_file(bucket, key, str(temp_target))
|
||
temp_target.replace(target)
|
||
except Exception as exc:
|
||
with contextlib.suppress(FileNotFoundError):
|
||
temp_target.unlink()
|
||
raise RuntimeError(f"下载 BOS 输入失败: {default_name}") from exc
|
||
return target
|
||
|
||
|
||
def signed_bos_source_url(raw_path: str) -> str:
|
||
bucket, key = parse_bos_path(raw_path)
|
||
if not bucket or not key:
|
||
return ""
|
||
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
if not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
|
||
return ""
|
||
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
||
return text_value(
|
||
client.generate_pre_signed_url(
|
||
bucket,
|
||
key,
|
||
expiration_in_seconds=3600,
|
||
protocol=scheme,
|
||
)
|
||
).strip()
|
||
|
||
|
||
def remote_video_url_for_source(source: dict[str, Any]) -> str:
|
||
preview_url = str(source.get("previewURL", "")).strip()
|
||
parsed_preview = urlparse(preview_url)
|
||
if parsed_preview.scheme.lower() in {"http", "https"}:
|
||
return preview_url
|
||
raw_path = str(source.get("path", "")).strip()
|
||
if raw_path.startswith("bos://"):
|
||
return signed_bos_source_url(raw_path)
|
||
return ""
|
||
|
||
|
||
def is_archive_path(path: pathlib.Path) -> bool:
|
||
name = path.name.lower()
|
||
return any(name.endswith(suffix) for suffix in ARCHIVE_FILE_SUFFIXES)
|
||
|
||
|
||
def safe_extract_archive(path: pathlib.Path, destination: pathlib.Path) -> list[str]:
|
||
destination.mkdir(parents=True, exist_ok=True)
|
||
destination_resolved = destination.resolve()
|
||
extracted_files: list[str] = []
|
||
|
||
def ensure_child(member_name: str) -> None:
|
||
target = (destination / member_name).resolve()
|
||
if target != destination_resolved and destination_resolved not in target.parents:
|
||
raise RuntimeError(f"archive entry escapes target directory: {member_name}")
|
||
|
||
if path.name.lower().endswith(".zip"):
|
||
with zipfile.ZipFile(path) as archive:
|
||
for member in archive.infolist():
|
||
if not member.filename or member.is_dir():
|
||
continue
|
||
ensure_child(member.filename)
|
||
extracted_files.append(member.filename)
|
||
archive.extractall(destination)
|
||
return extracted_files
|
||
|
||
with tarfile.open(path, "r:*") as archive:
|
||
members = archive.getmembers()
|
||
for member in members:
|
||
if not member.name or member.isdir():
|
||
continue
|
||
ensure_child(member.name)
|
||
extracted_files.append(member.name)
|
||
archive.extractall(destination)
|
||
return extracted_files
|
||
|
||
|
||
def artifact_slug(value: str) -> str:
|
||
normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-")
|
||
return normalized or "artifact"
|
||
|
||
|
||
def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str:
|
||
suffix = path.suffix.lower()
|
||
if suffix == ".json":
|
||
return "json"
|
||
if suffix in VIDEO_FILE_SUFFIXES:
|
||
return "video"
|
||
if suffix in IMAGE_FILE_SUFFIXES:
|
||
return "image"
|
||
return fallback
|
||
|
||
|
||
def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None:
|
||
cleaned = {key: json_compatible(value) for key, value in metadata.items() if value not in ("", None, [], {})}
|
||
return cleaned or None
|
||
|
||
|
||
def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None:
|
||
if path.suffix.lower() != ".json":
|
||
return None
|
||
for attempt in range(PAYLOAD_FILE_READ_ATTEMPTS):
|
||
if not path.exists():
|
||
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
|
||
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
|
||
continue
|
||
return None
|
||
try:
|
||
with path.open("r", encoding="utf-8") as handle:
|
||
value = json.load(handle)
|
||
except (FileNotFoundError, OSError, json.JSONDecodeError):
|
||
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
|
||
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
|
||
continue
|
||
return None
|
||
except Exception:
|
||
return None
|
||
if isinstance(value, dict):
|
||
return value
|
||
if isinstance(value, list):
|
||
return {"items": value, "itemCount": len(value)}
|
||
return {"value": value}
|
||
return None
|
||
|
||
|
||
def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None:
|
||
outputs = node_data.get("outputs", []) or []
|
||
if name:
|
||
for output in outputs:
|
||
if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name:
|
||
return output
|
||
if 0 <= index < len(outputs):
|
||
return outputs[index]
|
||
return None
|
||
|
||
|
||
def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]:
|
||
with path.open("rb") as handle:
|
||
raw = handle.read(PREVIEW_TEXT_LIMIT + 1)
|
||
truncated = len(raw) > PREVIEW_TEXT_LIMIT
|
||
text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace")
|
||
lines = text.splitlines()
|
||
if len(lines) > PREVIEW_TEXT_LINE_LIMIT:
|
||
text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT])
|
||
truncated = True
|
||
line_count = max(1, text.count("\n") + 1) if text else 0
|
||
return text, truncated, line_count
|
||
|
||
|
||
def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]:
|
||
text, truncated, line_count = read_text_sample(path)
|
||
return {
|
||
"preview": {
|
||
"kind": "text",
|
||
"format": format_name,
|
||
"text": text,
|
||
"truncated": truncated,
|
||
"lineCount": line_count,
|
||
}
|
||
}
|
||
|
||
|
||
def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]:
|
||
columns: list[str] = []
|
||
rows: list[dict[str, Any]] = []
|
||
total_rows = 0
|
||
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
|
||
reader = csv.reader(handle, delimiter=delimiter)
|
||
for index, row in enumerate(reader):
|
||
if index == 0:
|
||
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
|
||
continue
|
||
total_rows += 1
|
||
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
||
continue
|
||
width = max(len(columns), len(row))
|
||
row_payload: dict[str, Any] = {}
|
||
for column_index in range(width):
|
||
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
|
||
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
|
||
rows.append(row_payload)
|
||
return {
|
||
"preview": {
|
||
"kind": "table",
|
||
"format": format_name,
|
||
"columns": columns,
|
||
"rows": rows,
|
||
"rowCount": total_rows,
|
||
"truncated": total_rows > len(rows),
|
||
}
|
||
}
|
||
|
||
|
||
def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]:
|
||
columns: list[str] = []
|
||
rows: list[dict[str, Any]] = []
|
||
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
|
||
reader = csv.reader(handle, delimiter=delimiter)
|
||
for index, row in enumerate(reader):
|
||
if index == 0:
|
||
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
|
||
continue
|
||
width = max(len(columns), len(row))
|
||
row_payload: dict[str, Any] = {}
|
||
for column_index in range(width):
|
||
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
|
||
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
|
||
rows.append(row_payload)
|
||
return columns, rows
|
||
|
||
|
||
def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]:
|
||
rows: list[Any] = []
|
||
line_count = 0
|
||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||
for raw_line in handle:
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
line_count += 1
|
||
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
||
continue
|
||
try:
|
||
rows.append(json.loads(line))
|
||
except Exception:
|
||
rows.append({"value": line})
|
||
columns: list[str] = []
|
||
for row in rows:
|
||
if isinstance(row, dict):
|
||
for key in row.keys():
|
||
if key not in columns:
|
||
columns.append(key)
|
||
return {
|
||
"preview": {
|
||
"kind": "table",
|
||
"format": "jsonl",
|
||
"columns": columns,
|
||
"rows": rows,
|
||
"rowCount": line_count,
|
||
"truncated": line_count > len(rows),
|
||
}
|
||
}
|
||
|
||
|
||
def inspect_video_file(path: pathlib.Path) -> dict[str, Any]:
|
||
cv2, _ = require_cv2()
|
||
capture = cv2.VideoCapture(str(path))
|
||
if not capture.isOpened():
|
||
return {}
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
|
||
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
|
||
payload = clean_metadata(
|
||
{
|
||
"frameRate": round(fps, 3) if fps > 0 else None,
|
||
"frameCount": frame_count or None,
|
||
"resolution": f"{width}x{height}" if width > 0 and height > 0 else None,
|
||
"duration": round(duration, 3) if duration > 0 else None,
|
||
}
|
||
)
|
||
return payload or {}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]:
|
||
payload: dict[str, Any] = {
|
||
"preview": {
|
||
"kind": "image",
|
||
"format": path.suffix.lower().lstrip(".") or "image",
|
||
}
|
||
}
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
||
payload["preview"]["url"] = data_url_for_path(path, mime_type)
|
||
return payload
|
||
|
||
|
||
def preview_media_payload(path: pathlib.Path, mime_type: str, kind: str) -> dict[str, Any]:
|
||
preview: dict[str, Any] = {
|
||
"kind": kind,
|
||
"format": path.suffix.lower().lstrip(".") or kind,
|
||
}
|
||
if kind == "video":
|
||
preview.update(inspect_video_file(path))
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
||
preview["url"] = data_url_for_path(path, mime_type)
|
||
return {"preview": preview}
|
||
|
||
|
||
def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]:
|
||
try:
|
||
relative = candidate.relative_to(root).as_posix()
|
||
except ValueError:
|
||
relative = candidate.name
|
||
items.append({"path": relative, "sizeBytes": candidate.stat().st_size})
|
||
return items
|
||
|
||
|
||
def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]:
|
||
files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()]
|
||
extensions: dict[str, int] = {}
|
||
total_bytes = 0
|
||
for candidate in files:
|
||
extension = candidate.suffix.lower() or "<none>"
|
||
extensions[extension] = extensions.get(extension, 0) + 1
|
||
total_bytes += candidate.stat().st_size
|
||
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
|
||
return {
|
||
"root": str(root),
|
||
"fileCount": len(files),
|
||
"totalBytes": total_bytes,
|
||
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
|
||
"files": sample_file_entries(files, root),
|
||
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
|
||
}
|
||
|
||
|
||
def optional_json_file(path: pathlib.Path) -> Any:
|
||
if not path.exists():
|
||
return None
|
||
try:
|
||
with path.open("r", encoding="utf-8") as handle:
|
||
return json.load(handle)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def optional_jsonl_rows(path: pathlib.Path) -> list[Any]:
|
||
rows: list[Any] = []
|
||
if not path.exists():
|
||
return rows
|
||
try:
|
||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||
for raw_line in handle:
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
rows.append(json.loads(line))
|
||
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
||
break
|
||
except Exception:
|
||
return []
|
||
return rows
|
||
|
||
|
||
def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None:
|
||
candidates = [path] if path.is_dir() else [path.parent]
|
||
candidates.extend(list((path if path.is_dir() else path.parent).parents))
|
||
for candidate in candidates:
|
||
meta_dir = candidate / "meta"
|
||
if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")):
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]:
|
||
tree = summarize_directory_tree(root)
|
||
meta_dir = root / "meta"
|
||
info = optional_json_file(meta_dir / "info.json")
|
||
stats = optional_json_file(meta_dir / "stats.json")
|
||
tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl")
|
||
episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl")
|
||
parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()]
|
||
video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
|
||
return {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": "lerobot",
|
||
"root": str(root),
|
||
"summary": tree,
|
||
"info": info,
|
||
"stats": stats,
|
||
"tasks": tasks,
|
||
"episodes": episodes,
|
||
"parquetFiles": sample_file_entries(parquet_files, root),
|
||
"parquetCount": len(parquet_files),
|
||
"videoCount": len(video_files),
|
||
}
|
||
}
|
||
|
||
|
||
def sqlite_table_names(connection: sqlite3.Connection) -> list[str]:
|
||
rows = connection.execute(
|
||
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
||
).fetchall()
|
||
return [str(row[0]) for row in rows]
|
||
|
||
|
||
def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]:
|
||
connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro&immutable=1", uri=True)
|
||
try:
|
||
tables = sqlite_table_names(connection)
|
||
table_summaries: list[dict[str, Any]] = []
|
||
for table_name in tables[:12]:
|
||
count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0]
|
||
table_summaries.append({"name": table_name, "rowCount": int(count)})
|
||
payload: dict[str, Any] = {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": "sqlite",
|
||
"tables": table_summaries,
|
||
"tableCount": len(tables),
|
||
}
|
||
}
|
||
if "topics" in tables and "messages" in tables:
|
||
topic_rows = connection.execute(
|
||
"select id, name, type, serialization_format from topics order by id"
|
||
).fetchall()
|
||
message_counts = {
|
||
int(topic_id): int(count)
|
||
for topic_id, count in connection.execute(
|
||
"select topic_id, count(*) from messages group by topic_id order by topic_id"
|
||
).fetchall()
|
||
}
|
||
payload["preview"]["format"] = "rosbag2"
|
||
payload["preview"]["topics"] = [
|
||
{
|
||
"id": int(topic_id),
|
||
"name": str(name),
|
||
"type": str(topic_type),
|
||
"serialization": str(serialization),
|
||
"messageCount": message_counts.get(int(topic_id), 0),
|
||
}
|
||
for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT]
|
||
]
|
||
payload["preview"]["messageCount"] = int(sum(message_counts.values()))
|
||
return payload
|
||
finally:
|
||
connection.close()
|
||
|
||
|
||
def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]:
|
||
suffix = path.suffix.lower()
|
||
if suffix == ".db3":
|
||
return summarize_sqlite_payload(path)
|
||
header = ""
|
||
with path.open("rb") as handle:
|
||
header = handle.read(32).decode("utf-8", errors="replace")
|
||
format_name = "mcap" if suffix == ".mcap" else "rosbag1"
|
||
return {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": format_name,
|
||
"header": header.strip(),
|
||
"note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。",
|
||
}
|
||
}
|
||
|
||
|
||
def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None:
|
||
direct = params.get("SOURCE_PATH", "").strip()
|
||
if direct and pathlib.Path(direct).exists():
|
||
return pathlib.Path(direct)
|
||
for artifact in inputs:
|
||
payload = artifact.get("data")
|
||
if isinstance(payload, dict):
|
||
candidate = payload.get("sourcePath")
|
||
if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists():
|
||
return pathlib.Path(candidate)
|
||
source_path = artifact.get("path")
|
||
if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists():
|
||
return pathlib.Path(source_path)
|
||
return None
|
||
|
||
|
||
def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None:
|
||
data = artifact.get("data")
|
||
if not isinstance(data, dict):
|
||
return None
|
||
preview = data.get("preview")
|
||
if isinstance(preview, dict):
|
||
return preview
|
||
return None
|
||
|
||
|
||
def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]:
|
||
extensions: dict[str, int] = {}
|
||
files: list[dict[str, Any]] = []
|
||
for artifact in inputs:
|
||
name = str(artifact.get("name", "") or artifact.get("path", "")).strip()
|
||
extension = pathlib.Path(name).suffix.lower() or "<none>"
|
||
extensions[extension] = extensions.get(extension, 0) + 1
|
||
files.append(
|
||
{
|
||
"path": str(artifact.get("path", "")).strip() or name,
|
||
"name": name,
|
||
"size": str(artifact.get("size", "")),
|
||
}
|
||
)
|
||
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
|
||
return {
|
||
"fileCount": len(files),
|
||
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
|
||
"files": files[:PREVIEW_FILE_LIST_LIMIT],
|
||
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
|
||
}
|
||
|
||
|
||
def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str:
|
||
for artifact in inputs:
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
if not raw_path:
|
||
continue
|
||
parts = pathlib.PurePosixPath(raw_path).parts
|
||
for marker in ("meta", "data", "videos"):
|
||
if marker in parts:
|
||
index = parts.index(marker)
|
||
if index > 0:
|
||
return str(pathlib.PurePosixPath(*parts[:index]))
|
||
return ""
|
||
|
||
|
||
def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]:
|
||
info = None
|
||
stats = None
|
||
tasks: list[Any] = []
|
||
episodes: list[Any] = []
|
||
parquet_files: list[dict[str, Any]] = []
|
||
video_count = 0
|
||
for artifact in inputs:
|
||
name = str(artifact.get("name", "")).strip()
|
||
path = str(artifact.get("path", "")).strip()
|
||
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if name == "info.json" and isinstance(data, dict):
|
||
info = data
|
||
elif name == "stats.json" and isinstance(data, dict):
|
||
stats = data
|
||
elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list):
|
||
tasks = preview["rows"]
|
||
elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list):
|
||
episodes = preview["rows"]
|
||
if name.endswith(".parquet"):
|
||
parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))})
|
||
if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video":
|
||
video_count += 1
|
||
return {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": "lerobot",
|
||
"root": infer_lerobot_root_from_inputs(inputs),
|
||
"summary": summarize_input_artifacts(inputs),
|
||
"info": info,
|
||
"stats": stats,
|
||
"tasks": tasks,
|
||
"episodes": episodes,
|
||
"parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT],
|
||
"parquetCount": len(parquet_files),
|
||
"videoCount": video_count,
|
||
}
|
||
}
|
||
|
||
|
||
def parse_positive_float_param(params: dict[str, str], key: str, default: float) -> float:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
try:
|
||
parsed = float(raw)
|
||
except Exception as exc:
|
||
raise RuntimeError(f"{key} must be numeric") from exc
|
||
if not math.isfinite(parsed):
|
||
raise RuntimeError(f"{key} must be finite")
|
||
if parsed <= 0:
|
||
raise RuntimeError(f"{key} must be greater than 0")
|
||
return parsed
|
||
|
||
|
||
def parse_unit_interval_param(params: dict[str, str], key: str, default: float) -> float:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
try:
|
||
parsed = float(raw)
|
||
except Exception as exc:
|
||
raise RuntimeError(f"{key} must be numeric") from exc
|
||
if not math.isfinite(parsed):
|
||
raise RuntimeError(f"{key} must be finite")
|
||
if parsed < 0 or parsed > 1:
|
||
raise RuntimeError(f"{key} must be in the range [0, 1]")
|
||
return parsed
|
||
|
||
|
||
def float_value(value: Any, default: float) -> float:
|
||
if value is None:
|
||
return default
|
||
if isinstance(value, str) and not value.strip():
|
||
return default
|
||
try:
|
||
return float(value)
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def parse_positive_int_param(params: dict[str, str], key: str, default: int) -> int:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
if not EXACT_POSITIVE_INT_RE.fullmatch(raw):
|
||
if INTEGERISH_RE.fullmatch(raw):
|
||
try:
|
||
parsed_integerish = int(raw)
|
||
except Exception:
|
||
parsed_integerish = 0
|
||
if parsed_integerish <= 0:
|
||
raise RuntimeError(f"{key} must be greater than 0")
|
||
raise RuntimeError(f"{key} must be an exact positive integer")
|
||
raise RuntimeError(f"{key} must be an integer")
|
||
parsed = int(raw)
|
||
if parsed > MAX_SAFE_INTEGER:
|
||
raise RuntimeError(f"{key} must be a safe integer")
|
||
return parsed
|
||
|
||
|
||
def parse_bool_param(params: dict[str, str], key: str, default: bool = False) -> bool:
|
||
raw = params.get(key, "").strip().lower()
|
||
if not raw:
|
||
return default
|
||
if raw in TRUE_ENV_VALUES:
|
||
return True
|
||
if raw in FALSE_ENV_VALUES:
|
||
return False
|
||
raise RuntimeError(f"{key} must be a boolean")
|
||
|
||
|
||
def first_payload_with_keys(inputs: list[dict[str, Any]], *keys: str) -> dict[str, Any] | None:
|
||
for artifact in inputs:
|
||
payload = load_artifact_payload(artifact)
|
||
if isinstance(payload, dict) and all(key in payload for key in keys):
|
||
return payload
|
||
return None
|
||
|
||
|
||
def common_input_root(paths: list[str]) -> str:
|
||
filtered = [item for item in paths if item]
|
||
if not filtered:
|
||
return ""
|
||
if len(filtered) == 1:
|
||
return str(pathlib.Path(filtered[0]).parent)
|
||
try:
|
||
return os.path.commonpath(filtered)
|
||
except Exception:
|
||
return str(pathlib.Path(filtered[0]).parent)
|
||
|
||
|
||
def artifact_preview_url(artifact: dict[str, Any]) -> str:
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
value = preview.get("url")
|
||
return value if isinstance(value, str) else ""
|
||
|
||
|
||
def source_ref_from_artifact(artifact: dict[str, Any], relative_path: str = "") -> dict[str, Any]:
|
||
metadata = artifact.get("metadata") if isinstance(artifact.get("metadata"), dict) else {}
|
||
return clean_metadata(
|
||
{
|
||
"name": str(artifact.get("name", "")).strip(),
|
||
"path": str(artifact.get("path", "")).strip(),
|
||
"relativePath": relative_path,
|
||
"previewURL": artifact_preview_url(artifact),
|
||
"mimeType": str(metadata.get("mimeType", "")).strip(),
|
||
}
|
||
) or {}
|
||
|
||
|
||
def materialize_artifact_to_path(artifact: dict[str, Any], destination: pathlib.Path) -> bool:
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
if raw_path and pathlib.Path(raw_path).exists():
|
||
shutil.copy2(raw_path, destination)
|
||
return True
|
||
preview_url = artifact_preview_url(artifact)
|
||
decoded = decode_data_url(preview_url) if preview_url else None
|
||
if decoded is not None:
|
||
_, payload = decoded
|
||
destination.write_bytes(payload)
|
||
return True
|
||
payload = artifact.get("data")
|
||
if isinstance(payload, dict) and destination.suffix.lower() == ".json":
|
||
destination.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return True
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if preview.get("kind") == "text":
|
||
destination.write_text(str(preview.get("text", "")), encoding="utf-8")
|
||
return True
|
||
return False
|
||
|
||
|
||
def upload_relative_path(artifact: dict[str, Any], input_root: str, fallback_name: str) -> str:
|
||
explicit_relative_path = str(artifact.get("relativePath", "")).strip()
|
||
if explicit_relative_path:
|
||
return pathlib.PurePosixPath(explicit_relative_path).as_posix().lstrip("/")
|
||
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
payload_relative_path = str(payload.get("relativePath", "")).strip()
|
||
if payload_relative_path:
|
||
return pathlib.PurePosixPath(payload_relative_path).as_posix().lstrip("/")
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
relative_path = str(fallback_name or artifact.get("name", "")).strip() or "artifact"
|
||
if raw_path and input_root:
|
||
try:
|
||
relative_path = pathlib.Path(raw_path).relative_to(input_root).as_posix()
|
||
except Exception:
|
||
relative_path = pathlib.Path(raw_path).name or relative_path
|
||
if not relative_path:
|
||
relative_path = pathlib.Path(fallback_name or "artifact").name or "artifact"
|
||
return pathlib.PurePosixPath(relative_path).as_posix().lstrip("/")
|
||
|
||
|
||
def materialized_upload_source(artifact: dict[str, Any], staging_dir: pathlib.Path, relative_path: str) -> pathlib.Path | None:
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
if raw_path and os.path.exists(raw_path):
|
||
return pathlib.Path(raw_path)
|
||
destination = staging_dir / "_materialized_upload_inputs" / relative_path
|
||
if destination.exists():
|
||
return destination
|
||
if materialize_artifact_to_path(artifact, destination):
|
||
return destination
|
||
return None
|
||
|
||
|
||
def package_config_payload(item: dict[str, Any], manifest_payload: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"collectionId": item.get("collectionId", ""),
|
||
"meta": item.get("meta", {}),
|
||
"segmentDurationSeconds": manifest_payload.get("segmentDurationSeconds"),
|
||
"sourceVideo": item.get("sourceVideo", ""),
|
||
"startSecond": item.get("startSecond"),
|
||
"endSecond": item.get("endSecond"),
|
||
"duration": item.get("duration") or item.get("clipDuration"),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
}
|
||
|
||
|
||
def package_related_artifact_names(manifest_payload: dict[str, Any]) -> set[str]:
|
||
names: set[str] = set()
|
||
for item in manifest_payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
clip_id = str(item.get("clipId", "")).strip()
|
||
clip_name = str(item.get("clipName", "")).strip()
|
||
annotation_name = str(item.get("annotationFile", "")).strip() or (f"{clip_id}_annotation.json" if clip_id else "")
|
||
quality_name = f"{clip_id}_quality.json" if clip_id else ""
|
||
config_name = f"{clip_id}_config.json" if clip_id else ""
|
||
for candidate in (clip_name, annotation_name, quality_name, config_name):
|
||
if candidate:
|
||
names.add(candidate)
|
||
return names
|
||
|
||
|
||
def materialize_package_artifacts_from_manifest(
|
||
run_id: str,
|
||
node_id: str,
|
||
staging_dir: pathlib.Path,
|
||
manifest_payload: dict[str, Any],
|
||
node_data: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
archive_sources = manifest_payload.get("archiveSources", {}) if isinstance(manifest_payload.get("archiveSources"), dict) else {}
|
||
binding = output_binding(node_data)
|
||
materialized_artifacts: list[dict[str, Any]] = []
|
||
materialized_root = staging_dir / "_materialized_upload_inputs"
|
||
for item in manifest_payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
clip_id = str(item.get("clipId", "")).strip()
|
||
if not clip_id:
|
||
continue
|
||
package_name = str(item.get("packageName", "")).strip() or f"{clip_id}_package"
|
||
clip_name = str(item.get("clipName", "")).strip() or f"{clip_id}.mp4"
|
||
package_relative_dir = pathlib.PurePosixPath("packages") / package_name
|
||
package_dir = materialized_root / package_relative_dir
|
||
package_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
clip_destination = package_dir / clip_name
|
||
direct_clip_path = str(item.get("clipPath", "")).strip()
|
||
clip_source: pathlib.Path | None = pathlib.Path(direct_clip_path) if direct_clip_path and pathlib.Path(direct_clip_path).exists() else None
|
||
if clip_source is None:
|
||
clip_source = materialize_clip_from_source(item, staging_dir, archive_sources)
|
||
if clip_source is not None and clip_source.exists():
|
||
clip_size = clip_source.stat().st_size
|
||
clip_mime_type = mimetypes.guess_type(clip_name)[0] or "video/mp4"
|
||
clip_artifact = {
|
||
"id": f"{run_id}-{node_id}-{package_name}-{clip_name}",
|
||
"runId": run_id,
|
||
"nodeId": node_id,
|
||
"name": clip_name,
|
||
"type": "video",
|
||
"size": format_size(clip_size),
|
||
"path": str(clip_source),
|
||
"metadata": {"mimeType": clip_mime_type},
|
||
"data": {
|
||
"relativePath": (package_relative_dir / clip_name).as_posix(),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
},
|
||
}
|
||
clip_artifact["relativePath"] = (package_relative_dir / clip_name).as_posix()
|
||
materialized_artifacts.append(clip_artifact)
|
||
|
||
annotation_name = str(item.get("annotationFile", "")).strip() or f"{clip_id}_annotation.json"
|
||
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
|
||
annotation_artifact = write_json_artifact(run_id, node_id, package_dir, annotation_name, annotation_payload, output=binding)
|
||
annotation_artifact["relativePath"] = (package_relative_dir / annotation_name).as_posix()
|
||
materialized_artifacts.append(annotation_artifact)
|
||
|
||
quality_name = f"{clip_id}_quality.json"
|
||
quality_payload = {
|
||
"clip_id": clip_id,
|
||
"source_video": item.get("sourceVideo", ""),
|
||
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
|
||
}
|
||
quality_artifact = write_json_artifact(run_id, node_id, package_dir, quality_name, quality_payload, output=binding)
|
||
quality_artifact["relativePath"] = (package_relative_dir / quality_name).as_posix()
|
||
materialized_artifacts.append(quality_artifact)
|
||
|
||
config_name = f"{clip_id}_config.json"
|
||
config_artifact = write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
package_dir,
|
||
config_name,
|
||
package_config_payload(item, manifest_payload),
|
||
output=binding,
|
||
)
|
||
config_artifact["relativePath"] = (package_relative_dir / config_name).as_posix()
|
||
materialized_artifacts.append(config_artifact)
|
||
return materialized_artifacts
|
||
|
||
|
||
def materialize_input_tree(
|
||
inputs: list[dict[str, Any]],
|
||
output_dir: pathlib.Path,
|
||
) -> tuple[pathlib.Path | None, dict[str, dict[str, Any]]]:
|
||
paths = [str(item.get("path", "")).strip() for item in inputs if str(item.get("path", "")).strip()]
|
||
root = common_input_root(paths)
|
||
if not root:
|
||
return None, {}
|
||
target_root = output_dir / "_materialized_inputs"
|
||
if target_root.exists():
|
||
shutil.rmtree(target_root)
|
||
target_root.mkdir(parents=True, exist_ok=True)
|
||
source_refs: dict[str, dict[str, Any]] = {}
|
||
for index, artifact in enumerate(inputs):
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
relative_path = str(artifact.get("name", "")).strip() or f"artifact-{index}"
|
||
if raw_path:
|
||
try:
|
||
relative_path = pathlib.Path(raw_path).relative_to(root).as_posix()
|
||
except Exception:
|
||
relative_path = pathlib.Path(raw_path).name or relative_path
|
||
destination = target_root / relative_path
|
||
if materialize_artifact_to_path(artifact, destination):
|
||
source_refs[relative_path] = source_ref_from_artifact(artifact, relative_path)
|
||
if destination.is_file() and is_archive_path(destination):
|
||
extracted_files = safe_extract_archive(destination, target_root)
|
||
archive_preview_url = artifact_preview_url(artifact)
|
||
archive_key = artifact_slug(hashlib.sha256(archive_preview_url.encode("utf-8")).hexdigest()[:16]) if archive_preview_url else artifact_slug(relative_path)
|
||
for extracted_relative_path in extracted_files:
|
||
normalized_relative_path = pathlib.PurePosixPath(extracted_relative_path).as_posix().lstrip("/")
|
||
source_refs[normalized_relative_path] = clean_metadata(
|
||
{
|
||
"name": pathlib.PurePosixPath(normalized_relative_path).name,
|
||
"path": str(target_root / normalized_relative_path),
|
||
"relativePath": normalized_relative_path,
|
||
"archiveKey": archive_key,
|
||
"archivePreviewURL": archive_preview_url,
|
||
"archiveMemberPath": normalized_relative_path,
|
||
"archiveName": destination.name,
|
||
"mimeType": mimetypes.guess_type(normalized_relative_path)[0] or "",
|
||
}
|
||
) or {}
|
||
return target_root, source_refs
|
||
|
||
|
||
def detect_xplatform_collections(root: pathlib.Path) -> list[pathlib.Path]:
|
||
if not root.exists():
|
||
return []
|
||
candidates: list[pathlib.Path] = []
|
||
if (root / "meta.json").exists():
|
||
return [root]
|
||
for child in sorted(root.iterdir()):
|
||
if not child.is_dir():
|
||
continue
|
||
if (child / "meta.json").exists():
|
||
candidates.append(child)
|
||
continue
|
||
if any(candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES for candidate in child.rglob("*")):
|
||
candidates.append(child)
|
||
return candidates
|
||
|
||
|
||
def normalize_collection_relative_path(value: pathlib.PurePath | str) -> str:
|
||
text = value.as_posix() if isinstance(value, pathlib.PurePath) else str(value or "")
|
||
normalized = text.strip().strip("/")
|
||
if normalized in {"", "."}:
|
||
return ""
|
||
return normalized
|
||
|
||
|
||
def collection_name_from_source_path(source_path: str, fallback_name: str = "collection") -> str:
|
||
candidate = pathlib.PurePosixPath(str(source_path or "").rstrip("/")).name
|
||
return candidate or fallback_name
|
||
|
||
|
||
def parse_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
|
||
rows: dict[str, dict[str, Any]] = {}
|
||
if not isinstance(value, list):
|
||
return rows
|
||
for item in value:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
filename = str(item.get("filename", "")).strip()
|
||
if not filename:
|
||
continue
|
||
rows[filename] = item
|
||
return rows
|
||
|
||
|
||
def collection_file_entry(
|
||
root: pathlib.Path,
|
||
collection_dir: pathlib.Path,
|
||
candidate: pathlib.Path,
|
||
collection_id: str,
|
||
meta: dict[str, Any],
|
||
video_meta_rows: dict[str, dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
relative_path = candidate.relative_to(root).as_posix()
|
||
collection_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
|
||
collection_relative_file_path = candidate.relative_to(collection_dir).as_posix()
|
||
mime_type = mimetypes.guess_type(candidate.name)[0] or ""
|
||
entry: dict[str, Any] = {
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_dir.name,
|
||
"collectionPath": str(collection_dir),
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"relativePath": relative_path,
|
||
"path": str(candidate),
|
||
"sizeBytes": candidate.stat().st_size,
|
||
"mimeType": mime_type,
|
||
"source": {"path": str(candidate), "relativePath": relative_path, "mimeType": mime_type},
|
||
}
|
||
if candidate.suffix.lower() == ".json":
|
||
payload = optional_json_file(candidate)
|
||
if payload is not None:
|
||
entry["data"] = payload
|
||
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
matched_meta = video_meta_rows.get(candidate.name, {})
|
||
entry.update(
|
||
{
|
||
"videoFolder": collection_relative_file_path.split("/", 1)[0] if "/" in collection_relative_file_path else "",
|
||
"filename": candidate.name,
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
}
|
||
)
|
||
return entry
|
||
|
||
|
||
def collection_summary(
|
||
root: pathlib.Path,
|
||
collection_dir: pathlib.Path,
|
||
meta: dict[str, Any],
|
||
video_meta_rows: dict[str, dict[str, Any]],
|
||
) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
|
||
files = [candidate for candidate in sorted(collection_dir.rglob("*")) if candidate.is_file()]
|
||
videos = [candidate for candidate in files if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
|
||
total_bytes = sum(candidate.stat().st_size for candidate in files)
|
||
required = ["meta.json", "video_meta.json"]
|
||
missing_required = []
|
||
if not (collection_dir / "meta.json").exists():
|
||
missing_required.append("meta.json")
|
||
if not any((collection_dir / candidate).exists() for candidate in XPLATFORM_VIDEO_META_FILES):
|
||
missing_required.append("video_meta.json")
|
||
has_intrinsics = (collection_dir / "intrinsics.json").exists()
|
||
collection_root_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
|
||
collection_id = collection_key(collection_root_relative_path, collection_dir.name)
|
||
items: list[dict[str, Any]] = []
|
||
entries: list[dict[str, Any]] = []
|
||
for candidate in files:
|
||
entry = collection_file_entry(root, collection_dir, candidate, collection_id, meta, video_meta_rows)
|
||
entries.append(entry)
|
||
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
items.append(dict(entry))
|
||
summary = {
|
||
"id": collection_id,
|
||
"name": collection_dir.name,
|
||
"path": str(collection_dir),
|
||
"relativePath": collection_root_relative_path,
|
||
"meta": meta,
|
||
"hasIntrinsics": has_intrinsics,
|
||
"hasVideoMeta": bool(video_meta_rows),
|
||
"requiredFiles": required,
|
||
"missingRequiredFiles": missing_required,
|
||
"videoCount": len(videos),
|
||
"fileCount": len(files),
|
||
"totalBytes": total_bytes,
|
||
"valid": len(missing_required) == 0 and len(videos) > 0,
|
||
}
|
||
return summary, items, entries
|
||
|
||
|
||
def source_path_join(root: str, relative_path: str) -> str:
|
||
if not root:
|
||
return relative_path
|
||
return root.rstrip("/") + "/" + relative_path.lstrip("/")
|
||
|
||
|
||
def manifest_source_ref(entry: dict[str, Any], relative_path: str, source_path: str) -> dict[str, Any]:
|
||
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
|
||
storage_path = str(source.get("path", "") or entry.get("storagePath", "") or source_path_join(source_path, relative_path)).strip()
|
||
return clean_metadata(
|
||
{
|
||
"path": storage_path,
|
||
"relativePath": relative_path,
|
||
"previewURL": str(source.get("previewURL", "")).strip(),
|
||
"mimeType": str(entry.get("mimeType", "")).strip(),
|
||
}
|
||
) or {}
|
||
|
||
|
||
def manifest_entry_json_payload(entry: Any, source_path: str = "") -> Any:
|
||
if not isinstance(entry, dict):
|
||
return None
|
||
inline_data = entry.get("data")
|
||
if inline_data is not None:
|
||
return inline_data
|
||
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
|
||
candidates = [str(source.get("path", "")).strip(), str(entry.get("storagePath", "")).strip(), str(entry.get("path", "")).strip()]
|
||
for raw_candidate in candidates:
|
||
if not raw_candidate:
|
||
continue
|
||
if "://" in raw_candidate and not raw_candidate.startswith("file://"):
|
||
continue
|
||
candidate_text = raw_candidate[len("file://") :] if raw_candidate.startswith("file://") else raw_candidate
|
||
candidate_path = pathlib.Path(candidate_text)
|
||
if not candidate_path.is_absolute():
|
||
normalized_source_path = str(source_path or "").strip()
|
||
if normalized_source_path and "://" not in normalized_source_path:
|
||
candidate_path = pathlib.Path(normalized_source_path) / candidate_text
|
||
if candidate_path.is_file():
|
||
return optional_json_file(candidate_path)
|
||
return None
|
||
|
||
|
||
def manifest_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
|
||
if isinstance(value, dict) and isinstance(value.get("items"), list):
|
||
return parse_video_meta_rows(value.get("items"))
|
||
return parse_video_meta_rows(value)
|
||
|
||
|
||
def collection_key(relative_path: str, fallback_name: str = "collection") -> str:
|
||
normalized = relative_path.strip().strip("/")
|
||
if not normalized:
|
||
return artifact_slug(fallback_name)
|
||
return artifact_slug(normalized)
|
||
|
||
|
||
def fallback_manifest_collections(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
|
||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||
for entry in entries:
|
||
relative_path = str(entry.get("path", "")).strip().lstrip("/")
|
||
if not relative_path:
|
||
continue
|
||
parts = pathlib.PurePosixPath(relative_path).parts
|
||
if len(parts) < 2:
|
||
continue
|
||
grouped.setdefault(parts[0], []).append({**entry, "path": relative_path})
|
||
|
||
collection_summaries: list[dict[str, Any]] = []
|
||
items: list[dict[str, Any]] = []
|
||
indexed_entries: list[dict[str, Any]] = []
|
||
for collection_relative_path in sorted(grouped.keys()):
|
||
collection_entries = grouped[collection_relative_path]
|
||
collection_name = pathlib.PurePosixPath(collection_relative_path).name
|
||
collection_id = collection_key(collection_relative_path, collection_name)
|
||
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/meta.json"), None)
|
||
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/intrinsics.json"), None)
|
||
video_meta_entry = next(
|
||
(
|
||
entry
|
||
for entry in collection_entries
|
||
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
|
||
),
|
||
None,
|
||
)
|
||
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
|
||
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
|
||
meta = meta_payload if isinstance(meta_payload, dict) else {}
|
||
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
|
||
videos = [
|
||
entry
|
||
for entry in collection_entries
|
||
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
|
||
]
|
||
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
|
||
required = ["meta.json", "video_meta.json"]
|
||
missing_required = []
|
||
if meta_entry is None:
|
||
missing_required.append("meta.json")
|
||
if video_meta_entry is None:
|
||
missing_required.append("video_meta.json")
|
||
collection_path = source_path_join(source_path, collection_relative_path)
|
||
collection_summaries.append(
|
||
{
|
||
"id": collection_id,
|
||
"name": collection_name,
|
||
"path": collection_path,
|
||
"relativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"hasIntrinsics": intrinsics_entry is not None,
|
||
"hasVideoMeta": bool(video_meta_rows),
|
||
"requiredFiles": required,
|
||
"missingRequiredFiles": missing_required,
|
||
"videoCount": len(videos),
|
||
"fileCount": len(collection_entries),
|
||
"totalBytes": total_bytes,
|
||
"valid": len(missing_required) == 0 and len(videos) > 0,
|
||
}
|
||
)
|
||
for collection_entry in collection_entries:
|
||
relative_path = str(collection_entry.get("path", "")).strip()
|
||
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
|
||
indexed_entry = {
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
|
||
"mimeType": entry_mime_type,
|
||
"source": manifest_source_ref(collection_entry, relative_path, source_path),
|
||
}
|
||
if "data" in collection_entry:
|
||
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
|
||
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
indexed_entry.update(
|
||
{
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
}
|
||
)
|
||
indexed_entries.append(indexed_entry)
|
||
for video_entry in videos:
|
||
relative_path = str(video_entry.get("path", "")).strip()
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
items.append(
|
||
{
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
"source": manifest_source_ref(video_entry, relative_path, source_path),
|
||
}
|
||
)
|
||
return {
|
||
"datasetType": "x-platform",
|
||
"rootPath": source_path,
|
||
"collectionCount": len(collection_summaries),
|
||
"videoCount": len(items),
|
||
"collections": collection_summaries,
|
||
"items": items,
|
||
"entries": indexed_entries,
|
||
}
|
||
|
||
|
||
def scan_xplatform_manifest_entries(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
|
||
normalized_entries: list[dict[str, Any]] = []
|
||
meta_roots: set[str] = set()
|
||
auxiliary_roots: set[str] = set()
|
||
has_root_metadata = False
|
||
for entry in entries:
|
||
relative_path = str(entry.get("path", "")).strip().lstrip("/")
|
||
if not relative_path:
|
||
continue
|
||
normalized_entry = {**entry, "path": relative_path}
|
||
normalized_entries.append(normalized_entry)
|
||
path_obj = pathlib.PurePosixPath(relative_path)
|
||
parent = path_obj.parent.as_posix()
|
||
if parent in {".", ""}:
|
||
parent = ""
|
||
file_name = path_obj.name.lower()
|
||
if file_name == "meta.json":
|
||
if parent:
|
||
meta_roots.add(parent)
|
||
else:
|
||
has_root_metadata = True
|
||
elif is_xplatform_video_meta_name(file_name) or file_name == "intrinsics.json":
|
||
if parent:
|
||
auxiliary_roots.add(parent)
|
||
else:
|
||
has_root_metadata = True
|
||
|
||
root_candidates = meta_roots or auxiliary_roots
|
||
if has_root_metadata:
|
||
root_candidates = set(root_candidates)
|
||
root_candidates.add("")
|
||
if not root_candidates:
|
||
return fallback_manifest_collections(normalized_entries, source_path)
|
||
|
||
ordered_roots = sorted(root_candidates, key=lambda value: (-len(pathlib.PurePosixPath(value).parts), value))
|
||
grouped: dict[str, list[dict[str, Any]]] = {root: [] for root in ordered_roots}
|
||
unmatched_entries: list[dict[str, Any]] = []
|
||
for entry in normalized_entries:
|
||
relative_path = str(entry.get("path", "")).strip()
|
||
matched_root = next(
|
||
(
|
||
root
|
||
for root in ordered_roots
|
||
if (
|
||
(not root)
|
||
or relative_path == root
|
||
or relative_path.startswith(root + "/")
|
||
)
|
||
),
|
||
None,
|
||
)
|
||
if matched_root is not None:
|
||
grouped.setdefault(matched_root, []).append(entry)
|
||
else:
|
||
unmatched_entries.append(entry)
|
||
|
||
collection_summaries: list[dict[str, Any]] = []
|
||
items: list[dict[str, Any]] = []
|
||
indexed_entries: list[dict[str, Any]] = []
|
||
for collection_relative_path in sorted(grouped.keys()):
|
||
collection_entries = grouped[collection_relative_path]
|
||
if not collection_entries:
|
||
continue
|
||
collection_name = (
|
||
pathlib.PurePosixPath(collection_relative_path).name
|
||
if collection_relative_path
|
||
else collection_name_from_source_path(source_path)
|
||
)
|
||
collection_id = collection_key(collection_relative_path, collection_name)
|
||
meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json"
|
||
intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json"
|
||
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None)
|
||
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None)
|
||
video_meta_entry = next(
|
||
(
|
||
entry
|
||
for entry in collection_entries
|
||
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
|
||
),
|
||
None,
|
||
)
|
||
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
|
||
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
|
||
meta = meta_payload if isinstance(meta_payload, dict) else {}
|
||
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
|
||
videos = [
|
||
entry
|
||
for entry in collection_entries
|
||
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
|
||
]
|
||
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
|
||
required = ["meta.json", "video_meta.json"]
|
||
missing_required = []
|
||
if meta_entry is None:
|
||
missing_required.append("meta.json")
|
||
if video_meta_entry is None:
|
||
missing_required.append("video_meta.json")
|
||
collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path)
|
||
collection_summaries.append(
|
||
{
|
||
"id": collection_id,
|
||
"name": collection_name,
|
||
"path": collection_path,
|
||
"relativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"hasIntrinsics": intrinsics_entry is not None,
|
||
"hasVideoMeta": bool(video_meta_rows),
|
||
"requiredFiles": required,
|
||
"missingRequiredFiles": missing_required,
|
||
"videoCount": len(videos),
|
||
"fileCount": len(collection_entries),
|
||
"totalBytes": total_bytes,
|
||
"valid": len(missing_required) == 0 and len(videos) > 0,
|
||
}
|
||
)
|
||
for collection_entry in collection_entries:
|
||
relative_path = str(collection_entry.get("path", "")).strip()
|
||
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
|
||
indexed_entry = {
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
|
||
"mimeType": entry_mime_type,
|
||
"source": manifest_source_ref(collection_entry, relative_path, source_path),
|
||
}
|
||
if "data" in collection_entry:
|
||
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
|
||
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
indexed_entry.update(
|
||
{
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
}
|
||
)
|
||
indexed_entries.append(indexed_entry)
|
||
for video_entry in videos:
|
||
relative_path = str(video_entry.get("path", "")).strip()
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
items.append(
|
||
{
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
"source": manifest_source_ref(video_entry, relative_path, source_path),
|
||
}
|
||
)
|
||
if unmatched_entries:
|
||
fallback_payload = fallback_manifest_collections(unmatched_entries, source_path)
|
||
collection_summaries.extend(fallback_payload.get("collections", []))
|
||
items.extend(fallback_payload.get("items", []))
|
||
indexed_entries.extend(fallback_payload.get("entries", []))
|
||
return {
|
||
"datasetType": "x-platform",
|
||
"rootPath": source_path,
|
||
"collectionCount": len(collection_summaries),
|
||
"videoCount": len(items),
|
||
"collections": collection_summaries,
|
||
"items": items,
|
||
"entries": indexed_entries,
|
||
}
|
||
|
||
|
||
def with_xplatform_source_context(payload: dict[str, Any], source_manifest: dict[str, Any]) -> dict[str, Any]:
|
||
for key in (
|
||
"dataSource",
|
||
"sourcePath",
|
||
"scanMode",
|
||
"listedCount",
|
||
"newCount",
|
||
"sourceFingerprint",
|
||
"bucket",
|
||
"prefix",
|
||
"pattern",
|
||
"endpoint",
|
||
):
|
||
if key in source_manifest:
|
||
payload[key] = source_manifest.get(key)
|
||
return payload
|
||
|
||
|
||
def attach_xplatform_catalog_payload(payload: dict[str, Any], catalog_payload: dict[str, Any]) -> dict[str, Any]:
|
||
payload["catalogCollectionCount"] = int(catalog_payload.get("collectionCount", 0) or 0)
|
||
payload["catalogVideoCount"] = int(catalog_payload.get("videoCount", 0) or 0)
|
||
payload["catalogCollections"] = catalog_payload.get("collections", [])
|
||
payload["catalogItems"] = catalog_payload.get("items", [])
|
||
payload["catalogEntries"] = catalog_payload.get("entries", [])
|
||
return payload
|
||
|
||
|
||
def apply_source_refs_to_items(
|
||
items: list[dict[str, Any]],
|
||
source_refs: dict[str, dict[str, Any]],
|
||
) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]:
|
||
if not source_refs:
|
||
return items, {}
|
||
archive_sources: dict[str, dict[str, Any]] = {}
|
||
for item in items:
|
||
relative_path = str(item.get("relativePath", "")).strip()
|
||
if not relative_path:
|
||
continue
|
||
source_ref = source_refs.get(relative_path)
|
||
if source_ref:
|
||
archive_key = str(source_ref.get("archiveKey", "")).strip()
|
||
archive_preview_url = str(source_ref.get("archivePreviewURL", "")).strip()
|
||
if archive_key and archive_preview_url and archive_key not in archive_sources:
|
||
archive_sources[archive_key] = clean_metadata(
|
||
{
|
||
"name": str(source_ref.get("archiveName", "")).strip() or "archive",
|
||
"previewURL": archive_preview_url,
|
||
}
|
||
) or {}
|
||
item_source = {**source_ref, "path": item.get("path", "")}
|
||
item_source.pop("archivePreviewURL", None)
|
||
item_source.pop("archiveName", None)
|
||
item["source"] = item_source
|
||
return items, archive_sources
|
||
|
||
|
||
def scan_xplatform_dataset(root: pathlib.Path, source_refs: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]:
|
||
collections = detect_xplatform_collections(root)
|
||
collection_summaries: list[dict[str, Any]] = []
|
||
items: list[dict[str, Any]] = []
|
||
entries: list[dict[str, Any]] = []
|
||
for collection_dir in collections:
|
||
meta = optional_json_file(collection_dir / "meta.json")
|
||
meta = meta if isinstance(meta, dict) else {}
|
||
video_meta_rows, _ = load_collection_video_meta(collection_dir)
|
||
summary, collection_items, collection_entries = collection_summary(root, collection_dir, meta, video_meta_rows)
|
||
collection_summaries.append(summary)
|
||
items.extend(collection_items)
|
||
entries.extend(collection_entries)
|
||
items, archive_sources = apply_source_refs_to_items(items, source_refs or {})
|
||
entries, entry_archive_sources = apply_source_refs_to_items(entries, source_refs or {})
|
||
archive_sources.update(entry_archive_sources)
|
||
payload = {
|
||
"datasetType": "x-platform",
|
||
"rootPath": str(root),
|
||
"collectionCount": len(collection_summaries),
|
||
"videoCount": len(items),
|
||
"collections": collection_summaries,
|
||
"items": items,
|
||
"entries": entries,
|
||
}
|
||
if archive_sources:
|
||
payload["archiveSources"] = archive_sources
|
||
return payload
|
||
|
||
|
||
def load_xplatform_index_payload(
|
||
params: dict[str, str],
|
||
inputs: list[dict[str, Any]],
|
||
output_dir: pathlib.Path,
|
||
) -> dict[str, Any]:
|
||
source_manifest = first_payload_with_keys(inputs, "entries")
|
||
if source_manifest is not None:
|
||
entries = source_manifest.get("entries")
|
||
if isinstance(entries, list):
|
||
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
|
||
payload = scan_xplatform_manifest_entries(entries, manifest_source_path)
|
||
catalog_entries = source_manifest.get("catalogEntries")
|
||
if isinstance(catalog_entries, list):
|
||
catalog_payload = scan_xplatform_manifest_entries(catalog_entries, manifest_source_path)
|
||
attach_xplatform_catalog_payload(payload, catalog_payload)
|
||
return with_xplatform_source_context(payload, source_manifest)
|
||
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
|
||
if manifest_source_path and pathlib.Path(manifest_source_path).exists():
|
||
root = pathlib.Path(manifest_source_path)
|
||
return scan_xplatform_dataset(root if root.is_dir() else root.parent)
|
||
for artifact in inputs:
|
||
name = str(artifact.get("name", "")).strip().lower()
|
||
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
if name != "source-manifest.json" and not any(key in data for key in ("entriesCount", "catalogEntriesCount", "sourcePath")):
|
||
continue
|
||
raise RuntimeError("X 平台数据集索引未能读取 source-manifest 的 entries,拒绝回退到空目录扫描")
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
root = source_path if source_path.is_dir() else source_path.parent
|
||
return scan_xplatform_dataset(root)
|
||
materialized_root, source_refs = materialize_input_tree(inputs, output_dir)
|
||
if materialized_root is not None:
|
||
return scan_xplatform_dataset(materialized_root, source_refs)
|
||
raise RuntimeError("X 平台数据集索引需要 SOURCE_PATH 或上游输入提供真实数据目录")
|
||
|
||
|
||
def normalize_date_value(value: Any) -> str:
|
||
text = str(value or "").strip()
|
||
digits = "".join(character for character in text if character.isdigit())
|
||
if len(digits) >= 8:
|
||
return digits[:8]
|
||
return ""
|
||
|
||
|
||
def date_in_range(value: Any, start: str, end: str) -> bool:
|
||
normalized = normalize_date_value(value)
|
||
if not normalized:
|
||
return False
|
||
if start and normalized < start:
|
||
return False
|
||
if end and normalized > end:
|
||
return False
|
||
return True
|
||
|
||
|
||
def materialize_archive_member(
|
||
source: dict[str, Any],
|
||
archive_sources: dict[str, Any],
|
||
default_name: str,
|
||
output_dir: pathlib.Path,
|
||
) -> pathlib.Path | None:
|
||
archive_key = str(source.get("archiveKey", "")).strip()
|
||
archive_source = archive_sources.get(archive_key) if isinstance(archive_sources, dict) else {}
|
||
archive_source = archive_source if isinstance(archive_source, dict) else {}
|
||
preview_url = str(archive_source.get("previewURL", "")).strip()
|
||
member_path = str(source.get("archiveMemberPath", "")).strip()
|
||
decoded = decode_data_url(preview_url) if preview_url and member_path else None
|
||
if decoded is None:
|
||
return None
|
||
_, payload = decoded
|
||
archive_name = pathlib.PurePosixPath(str(archive_source.get("name", "")).strip() or "archive").name
|
||
archive_path = output_dir / "_materialized_sources" / archive_name
|
||
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||
archive_path.write_bytes(payload)
|
||
destination = output_dir / "_materialized_sources" / default_name
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
if archive_name.lower().endswith(".zip"):
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
with archive.open(member_path) as handle:
|
||
destination.write_bytes(handle.read())
|
||
return destination
|
||
with tarfile.open(archive_path, "r:*") as archive:
|
||
extracted = archive.extractfile(member_path)
|
||
if extracted is None:
|
||
return None
|
||
with extracted:
|
||
destination.write_bytes(extracted.read())
|
||
return destination
|
||
|
||
|
||
def source_path_from_item(
|
||
item: dict[str, Any],
|
||
default_name: str,
|
||
output_dir: pathlib.Path,
|
||
archive_sources: dict[str, Any] | None = None,
|
||
) -> pathlib.Path:
|
||
source = item.get("source") if isinstance(item.get("source"), dict) else {}
|
||
direct = str(source.get("path", "") or item.get("path", "")).strip()
|
||
if direct and pathlib.Path(direct).exists():
|
||
return pathlib.Path(direct)
|
||
if direct.startswith("bos://"):
|
||
cached_target = workspace_cached_bos_source_path(direct, output_dir)
|
||
if cached_target is not None and cached_target.exists() and cached_target.stat().st_size > 0:
|
||
return cached_target
|
||
cached_bos_source = download_bos_source(direct, default_name, output_dir)
|
||
if cached_bos_source is not None:
|
||
return cached_bos_source
|
||
preview_url = str(source.get("previewURL", "")).strip()
|
||
if cached_target is not None and preview_url:
|
||
cached_remote_source = download_remote_source_to_path(preview_url, cached_target)
|
||
if cached_remote_source is not None:
|
||
return cached_remote_source
|
||
archive_member = materialize_archive_member(source, archive_sources or {}, default_name, output_dir)
|
||
if archive_member is not None:
|
||
return archive_member
|
||
preview_url = str(source.get("previewURL", "")).strip()
|
||
remote_path = download_remote_source(preview_url, default_name, output_dir) if preview_url else None
|
||
if remote_path is not None:
|
||
return remote_path
|
||
decoded = decode_data_url(preview_url) if preview_url else None
|
||
if decoded is None:
|
||
raise RuntimeError(f"无法获取真实输入文件: {default_name}")
|
||
_, payload = decoded
|
||
path = output_dir / "_materialized_sources" / default_name
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(payload)
|
||
return path
|
||
|
||
|
||
def clip_segment_bounds(item: dict[str, Any]) -> tuple[float, float]:
|
||
start_second = float(item.get("startSecond") or 0.0)
|
||
end_second = float(item.get("endSecond") or 0.0)
|
||
duration = float(item.get("duration") or 0.0)
|
||
if end_second <= start_second and duration > 0:
|
||
end_second = start_second + duration
|
||
if end_second <= start_second:
|
||
end_second = start_second + 0.1
|
||
return start_second, end_second
|
||
|
||
|
||
def materialize_clip_from_source(
|
||
item: dict[str, Any],
|
||
output_dir: pathlib.Path,
|
||
archive_sources: dict[str, Any] | None = None,
|
||
) -> pathlib.Path | None:
|
||
source = item.get("source") if isinstance(item.get("source"), dict) else {}
|
||
source_name = (
|
||
str(item.get("sourceVideo", "")).strip()
|
||
or pathlib.PurePosixPath(str(source.get("path", "")).strip()).name
|
||
or "source.mp4"
|
||
)
|
||
clip_name = str(item.get("clipName", "")).strip() or f"{pathlib.Path(source_name).stem}_clip.mp4"
|
||
destination = output_dir / "_materialized_clips" / clip_name
|
||
if destination.exists():
|
||
return destination
|
||
start_second, end_second = clip_segment_bounds(item)
|
||
remote_source_url = remote_video_url_for_source(source) if source else ""
|
||
if remote_source_url:
|
||
try:
|
||
write_video_segment_from_remote_url(remote_source_url, destination, start_second, end_second)
|
||
return destination
|
||
except RuntimeError as exc:
|
||
log(f"[operator-runner][warn] fallback to temporary source materialization for {clip_name}: {exc}")
|
||
if not source:
|
||
return None
|
||
source_path = source_path_from_item({"source": source}, source_name, output_dir, archive_sources)
|
||
write_video_segment(source_path, destination, start_second, end_second)
|
||
return destination
|
||
|
||
|
||
def analyze_video_quality_metrics(path: pathlib.Path, clarity_scale: float) -> dict[str, Any]:
|
||
cv2, np = require_cv2()
|
||
capture = cv2.VideoCapture(str(path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频文件: {path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
|
||
positions = sampled_frame_positions(frame_count)
|
||
if not positions:
|
||
positions = [0]
|
||
clarities: list[float] = []
|
||
shifts: list[tuple[float, float]] = []
|
||
skin_ratios: list[float] = []
|
||
previous_gray = None
|
||
previous_position: int | None = None
|
||
analysis_diagonal = math.sqrt((width or 1) ** 2 + (height or 1) ** 2)
|
||
for position in positions:
|
||
if frame_count > 0:
|
||
capture.set(cv2.CAP_PROP_POS_FRAMES, position)
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
continue
|
||
analysis_frame = resize_quality_frame(frame)
|
||
analysis_height, analysis_width = analysis_frame.shape[:2]
|
||
analysis_diagonal = math.sqrt((analysis_width or 1) ** 2 + (analysis_height or 1) ** 2)
|
||
gray = cv2.cvtColor(analysis_frame, cv2.COLOR_BGR2GRAY)
|
||
laplacian = cv2.Laplacian(gray, cv2.CV_64F).var()
|
||
clarities.append(float(laplacian))
|
||
if previous_gray is not None:
|
||
gap = max(position - previous_position, 1) if previous_position is not None else 1
|
||
shift, _ = cv2.phaseCorrelate(np.float32(previous_gray), np.float32(gray))
|
||
shifts.append((float(shift[0]) / gap, float(shift[1]) / gap))
|
||
mask = build_skin_mask(analysis_frame)
|
||
skin_ratios.append(largest_skin_component_ratio(mask))
|
||
previous_gray = gray
|
||
previous_position = position
|
||
clarity_raw = sum(clarities) / max(len(clarities), 1)
|
||
clarity = 1.0 - math.exp(-max(clarity_raw, 0.0) / clarity_scale)
|
||
shake = motion_jitter_ratio(shifts, analysis_diagonal) if shifts else 0.0
|
||
hand_occlusion = sum(skin_ratios) / max(len(skin_ratios), 1) if skin_ratios else 0.0
|
||
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
|
||
quality_score = max(0.0, min(1.0, (clarity + (1.0 - min(shake, 1.0)) + (1.0 - min(hand_occlusion, 1.0))) / 3.0))
|
||
return {
|
||
"codec": "opencv",
|
||
"resolution": f"{width}x{height}" if width > 0 and height > 0 else "",
|
||
"frameRate": round(fps, 3) if fps > 0 else None,
|
||
"duration": round(duration, 3) if duration > 0 else None,
|
||
"clarity": round(clarity, 4),
|
||
"shake": round(shake, 4),
|
||
"handOcclusion": round(hand_occlusion, 4),
|
||
"qualityScore": round(quality_score, 4),
|
||
"sampleCount": len(clarities),
|
||
}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def stable_sample_hit(value: str, percent: float) -> bool:
|
||
if percent >= 100:
|
||
return True
|
||
if percent <= 0:
|
||
return False
|
||
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||
score = int(digest[:8], 16) / 0xFFFFFFFF
|
||
return score <= (percent / 100.0)
|
||
|
||
|
||
def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None:
|
||
suffix = path.suffix.lower()
|
||
if suffix in TABULAR_FILE_SUFFIXES:
|
||
return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip("."))
|
||
if suffix == ".jsonl":
|
||
return preview_jsonl_payload(path)
|
||
if suffix in TEXT_FILE_SUFFIXES:
|
||
format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".")
|
||
return preview_text_payload(path, format_name)
|
||
if suffix in IMAGE_FILE_SUFFIXES:
|
||
return preview_image_payload(path, mime_type)
|
||
if suffix in ROSBAG_FILE_SUFFIXES:
|
||
return summarize_rosbag_file(path)
|
||
if suffix == ".pdf":
|
||
return {"preview": {"kind": "document", "format": "pdf"}}
|
||
if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES:
|
||
return preview_media_payload(path, mime_type or "video/mp4", "video")
|
||
if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
|
||
return preview_media_payload(path, mime_type or "audio/mpeg", "audio")
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
||
return {"preview": {"kind": "binary", "format": suffix.lstrip(".") or "file", "url": data_url_for_path(path, mime_type)}}
|
||
return None
|
||
|
||
|
||
def build_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
path: pathlib.Path,
|
||
artifact_type: str,
|
||
data: dict[str, Any] | None = None,
|
||
metadata: dict[str, Any] | None = None,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
mime_type, _ = mimetypes.guess_type(path.name)
|
||
size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8"))
|
||
sha256_value = ""
|
||
sha256_metadata: dict[str, Any] = {}
|
||
if path.exists():
|
||
sha256_value, sha256_metadata = artifact_sha256(path, size_bytes)
|
||
artifact_metadata = clean_metadata(
|
||
{
|
||
**(metadata or {}),
|
||
**sha256_metadata,
|
||
"sha256": sha256_value,
|
||
"mimeType": mime_type or "",
|
||
}
|
||
)
|
||
artifact = {
|
||
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}",
|
||
"runId": run_id,
|
||
"nodeId": node_id,
|
||
"name": path.name,
|
||
"type": artifact_type,
|
||
"size": format_size(size_bytes),
|
||
"path": str(path),
|
||
}
|
||
if output:
|
||
if str(output.get("id", "")).strip():
|
||
artifact["portId"] = str(output.get("id", "")).strip()
|
||
if str(output.get("name", "")).strip():
|
||
artifact["portName"] = str(output.get("name", "")).strip()
|
||
if artifact_metadata:
|
||
artifact["metadata"] = artifact_metadata
|
||
if data:
|
||
artifact["data"] = data
|
||
return artifact
|
||
|
||
|
||
def build_virtual_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
name: str,
|
||
path: str,
|
||
artifact_type: str,
|
||
size_bytes: int,
|
||
data: dict[str, Any] | None = None,
|
||
metadata: dict[str, Any] | None = None,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
artifact = {
|
||
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else name)}-{artifact_slug(name)}",
|
||
"runId": run_id,
|
||
"nodeId": node_id,
|
||
"name": name,
|
||
"type": artifact_type,
|
||
"size": format_size(max(0, size_bytes)),
|
||
"path": path,
|
||
}
|
||
if output:
|
||
if str(output.get("id", "")).strip():
|
||
artifact["portId"] = str(output.get("id", "")).strip()
|
||
if str(output.get("name", "")).strip():
|
||
artifact["portName"] = str(output.get("name", "")).strip()
|
||
if metadata:
|
||
cleaned = clean_metadata(metadata)
|
||
if cleaned:
|
||
artifact["metadata"] = cleaned
|
||
if data:
|
||
artifact["data"] = data
|
||
return artifact
|
||
|
||
|
||
def dataset_partition_keys(payload: Any) -> list[str]:
|
||
if not isinstance(payload, dict):
|
||
return []
|
||
keys: list[str] = []
|
||
for key in DATASET_PARTITION_KEYS:
|
||
if isinstance(payload.get(key), list):
|
||
keys.append(key)
|
||
return keys
|
||
|
||
|
||
def should_externalize_structured_payload(payload: Any) -> bool:
|
||
if not isinstance(payload, dict):
|
||
return False
|
||
if dataset_partition_keys(payload):
|
||
return True
|
||
dataset_type = str(payload.get("datasetType", "")).strip()
|
||
if dataset_type:
|
||
return True
|
||
return False
|
||
|
||
|
||
def dataset_payload_summary(payload: Any) -> dict[str, Any]:
|
||
payload = normalize_json_payload(payload)
|
||
summary: dict[str, Any] = {}
|
||
for key in (
|
||
"datasetType",
|
||
"count",
|
||
"itemCount",
|
||
"collectionCount",
|
||
"entryCount",
|
||
"videoCount",
|
||
"clipCount",
|
||
"annotationCount",
|
||
"packageCount",
|
||
"sampledCount",
|
||
"passedCount",
|
||
"failedCount",
|
||
"validCollectionCount",
|
||
"invalidCollectionCount",
|
||
"dataSource",
|
||
"sourcePath",
|
||
"bucket",
|
||
"prefix",
|
||
"endpoint",
|
||
"pattern",
|
||
"scanMode",
|
||
"sourceFingerprint",
|
||
"listedCount",
|
||
"newCount",
|
||
"rootPath",
|
||
"segmentDurationSeconds",
|
||
):
|
||
if key in payload:
|
||
summary[key] = json_compatible(payload[key])
|
||
if "preview" in payload:
|
||
summary["preview"] = json_compatible(payload.get("preview"))
|
||
for key in dataset_partition_keys(payload):
|
||
value = payload.get(key)
|
||
if isinstance(value, list):
|
||
summary[f"{key}Count"] = len(value)
|
||
if "preview" not in summary:
|
||
summary["preview"] = {
|
||
"kind": "dataset",
|
||
"format": str(payload.get("datasetType", "")).strip() or "json",
|
||
"counts": {key: summary[key] for key in summary.keys() if key.endswith("Count") or key == "count"},
|
||
}
|
||
return summary
|
||
|
||
|
||
def chunk_list(values: list[Any], chunk_size: int = DATASET_PARTITION_SIZE) -> list[list[Any]]:
|
||
if chunk_size <= 0:
|
||
chunk_size = DATASET_PARTITION_SIZE
|
||
return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] or [[]]
|
||
|
||
|
||
def externalize_structured_payload(path: pathlib.Path, payload: dict[str, Any]) -> dict[str, Any]:
|
||
normalized_payload = json_compatible(payload)
|
||
dataset_dir = path.parent / "_datasets" / artifact_slug(path.stem or path.name)
|
||
partitions_dir = dataset_dir / "partitions"
|
||
partitions_dir.mkdir(parents=True, exist_ok=True)
|
||
partition_files: dict[str, list[str]] = {}
|
||
partition_count = 0
|
||
partition_keys = dataset_partition_keys(normalized_payload)
|
||
for key in partition_keys:
|
||
values = normalized_payload.get(key)
|
||
if not isinstance(values, list):
|
||
continue
|
||
files: list[str] = []
|
||
for index, chunk in enumerate(chunk_list(values), start=1):
|
||
partition_path = partitions_dir / f"{key}-p{index:06d}.json"
|
||
write_json_file(partition_path, chunk)
|
||
files.append(str(partition_path))
|
||
partition_files[key] = files
|
||
partition_count += len(files)
|
||
manifest = {
|
||
"kind": "dataset",
|
||
"datasetType": str(normalized_payload.get("datasetType", "")).strip() or "json",
|
||
"payloadPath": str(path),
|
||
"partitionDir": str(partitions_dir),
|
||
"partitionKeys": partition_keys,
|
||
"partitionFiles": partition_files,
|
||
"summary": dataset_payload_summary(normalized_payload),
|
||
"generatedAt": now_iso(),
|
||
}
|
||
manifest_path = write_json_file(dataset_dir / "manifest.json", manifest)
|
||
data_ref = {
|
||
"kind": "dataset",
|
||
"datasetType": manifest["datasetType"],
|
||
"manifestPath": str(manifest_path),
|
||
"payloadPath": str(path),
|
||
"partitionDir": str(partitions_dir),
|
||
"partitionCount": partition_count,
|
||
}
|
||
item_count = 0
|
||
for key in partition_keys:
|
||
values = normalized_payload.get(key)
|
||
if isinstance(values, list):
|
||
item_count += len(values)
|
||
if item_count > 0:
|
||
data_ref["itemCount"] = item_count
|
||
summary = dataset_payload_summary(normalized_payload)
|
||
summary["dataRef"] = data_ref
|
||
return summary
|
||
|
||
|
||
def write_json_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
filename: str,
|
||
payload: Any,
|
||
artifact_type: str = "json",
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
normalized_payload = json_compatible(payload)
|
||
path = output_dir / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8") as handle:
|
||
json.dump(normalized_payload, handle, ensure_ascii=False, indent=2)
|
||
artifact_data: dict[str, Any] | None = normalize_json_payload(normalized_payload)
|
||
if isinstance(normalized_payload, dict) and should_externalize_structured_payload(normalized_payload):
|
||
artifact_data = externalize_structured_payload(path, normalized_payload)
|
||
return build_artifact(run_id, node_id, path, artifact_type, data=artifact_data, output=output)
|
||
|
||
|
||
def summarize_json_like(value: Any) -> dict[str, Any]:
|
||
if isinstance(value, dict):
|
||
return {"kind": "object", "keys": sorted(value.keys())}
|
||
if isinstance(value, list):
|
||
return {"kind": "list", "length": len(value)}
|
||
return {"kind": type(value).__name__, "value": value}
|
||
|
||
|
||
def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]:
|
||
data = artifact.get("data")
|
||
if isinstance(data, dict):
|
||
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
|
||
payload_path = str(data_ref.get("payloadPath", "")).strip()
|
||
if payload_path:
|
||
payload = load_payload_from_file(pathlib.Path(payload_path))
|
||
if payload is not None:
|
||
return normalize_json_payload(payload)
|
||
if isinstance(data, dict):
|
||
return data
|
||
path_value = artifact.get("path")
|
||
if isinstance(path_value, str) and path_value:
|
||
payload = load_payload_from_file(pathlib.Path(path_value))
|
||
if payload is not None:
|
||
return normalize_json_payload(payload)
|
||
summary = {
|
||
"name": artifact.get("name", ""),
|
||
"type": artifact.get("type", ""),
|
||
"path": artifact.get("path", ""),
|
||
"size": artifact.get("size", ""),
|
||
}
|
||
return {key: value for key, value in summary.items() if value}
|
||
|
||
|
||
def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]:
|
||
merged: dict[str, Any] = {}
|
||
for value in values:
|
||
merged.update(value)
|
||
return merged
|
||
|
||
|
||
def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]:
|
||
entries: list[dict[str, Any]] = []
|
||
if not base_path.exists():
|
||
return entries
|
||
if base_path.is_file():
|
||
relative = base_path.name
|
||
if pattern and not fnmatch.fnmatch(relative, pattern):
|
||
return entries
|
||
return [
|
||
{
|
||
"path": relative,
|
||
"sizeBytes": base_path.stat().st_size,
|
||
"sha256": sha256_file(base_path),
|
||
}
|
||
]
|
||
for candidate in sorted(base_path.rglob("*")):
|
||
if candidate.is_dir():
|
||
continue
|
||
relative = candidate.relative_to(base_path).as_posix()
|
||
if pattern and not fnmatch.fnmatch(relative, pattern):
|
||
continue
|
||
entries.append(
|
||
{
|
||
"path": relative,
|
||
"sizeBytes": candidate.stat().st_size,
|
||
"sha256": sha256_file(candidate),
|
||
}
|
||
)
|
||
return entries
|
||
|
||
|
||
def list_bos_entries(
|
||
bucket: str,
|
||
prefix: str,
|
||
pattern: str,
|
||
endpoint: str,
|
||
access_key_id: str,
|
||
secret_access_key: str,
|
||
) -> list[dict[str, Any]]:
|
||
client, scheme = build_bos_client(endpoint, access_key_id, secret_access_key)
|
||
normalized_prefix = bos_prefix(prefix)
|
||
prefix_with_slash = f"{normalized_prefix}/" if normalized_prefix else ""
|
||
entries: list[dict[str, Any]] = []
|
||
for candidate in client.list_all_objects(bucket, prefix=normalized_prefix or None):
|
||
key = text_value(getattr(candidate, "key", "")).strip().lstrip("/")
|
||
if not key or key.endswith("/"):
|
||
continue
|
||
if normalized_prefix:
|
||
if key == normalized_prefix:
|
||
continue
|
||
if not key.startswith(prefix_with_slash):
|
||
continue
|
||
relative_path = key[len(prefix_with_slash):]
|
||
else:
|
||
relative_path = key
|
||
if not relative_path:
|
||
continue
|
||
if pattern and not fnmatch.fnmatch(relative_path, pattern):
|
||
continue
|
||
size_bytes = int(getattr(candidate, "size", 0) or 0)
|
||
mime_type = mimetypes.guess_type(relative_path)[0] or ""
|
||
storage_path = canonical_bos_path(bucket, key)
|
||
entry: dict[str, Any] = {
|
||
"path": relative_path,
|
||
"storagePath": storage_path,
|
||
"sizeBytes": size_bytes,
|
||
"mimeType": mime_type,
|
||
"etag": text_value(getattr(candidate, "e_tag", "") or getattr(candidate, "etag", "")),
|
||
"versionId": text_value(getattr(candidate, "version_id", "") or getattr(candidate, "versionId", "")),
|
||
"lastModified": text_value(getattr(candidate, "last_modified", "") or getattr(candidate, "lastModified", "")),
|
||
"source": {"path": storage_path},
|
||
}
|
||
file_name = pathlib.PurePosixPath(relative_path).name.lower()
|
||
should_inline_json = relative_path.lower().endswith(".json") and (
|
||
file_name in XPLATFORM_INLINE_JSON_FILES or size_bytes <= PREVIEW_INLINE_BINARY_LIMIT
|
||
)
|
||
if should_inline_json:
|
||
raw = client.get_object_as_string(bucket, key)
|
||
payload_text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
|
||
try:
|
||
entry["data"] = json.loads(payload_text)
|
||
except Exception:
|
||
entry["data"] = {"value": payload_text}
|
||
elif pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
|
||
entry["source"]["previewURL"] = client.generate_pre_signed_url(
|
||
bucket,
|
||
key,
|
||
expiration_in_seconds=3600,
|
||
protocol=scheme,
|
||
)
|
||
entries.append(clean_metadata(entry) or {})
|
||
return entries
|
||
|
||
|
||
def source_entry_fingerprint(data_source: str, bucket: str, entry: dict[str, Any]) -> str:
|
||
normalized_source = (data_source or "BOS").strip().upper() or "BOS"
|
||
relative_path = str(entry.get("path", "")).strip()
|
||
version_id = str(entry.get("versionId", "")).strip()
|
||
sha256 = str(entry.get("sha256", "")).strip()
|
||
etag = str(entry.get("etag", "")).strip()
|
||
size_bytes = str(entry.get("sizeBytes", "")).strip()
|
||
last_modified = str(entry.get("lastModified", "")).strip()
|
||
parts = [normalized_source, str(bucket or "").strip(), relative_path]
|
||
if version_id:
|
||
parts.append(f"version:{version_id}")
|
||
elif sha256:
|
||
parts.append(f"sha256:{sha256}")
|
||
else:
|
||
parts.extend([f"etag:{etag}", f"size:{size_bytes}", f"lastModified:{last_modified}"])
|
||
raw = "\n".join(parts).encode("utf-8")
|
||
return hashlib.sha256(raw).hexdigest()
|
||
|
||
|
||
def incremental_unit(params: dict[str, str]) -> str:
|
||
value = (params.get("INCREMENTAL_UNIT") or "").strip().upper()
|
||
if not value:
|
||
return "OBJECT"
|
||
if value in {"OBJECT", "DIRECTORY"}:
|
||
return value
|
||
raise RuntimeError("INCREMENTAL_UNIT must be OBJECT or DIRECTORY")
|
||
|
||
|
||
def incremental_directory_depth(params: dict[str, str]) -> int:
|
||
return parse_positive_int_param(params, "INCREMENTAL_DIRECTORY_DEPTH", 1)
|
||
|
||
|
||
def directory_incremental_key(relative_path: str, depth: int) -> str:
|
||
clean_relative = str(relative_path or "").strip().lstrip("/")
|
||
if not clean_relative:
|
||
return "__root__"
|
||
parts = [segment for segment in pathlib.PurePosixPath(clean_relative).parts if segment not in ("", ".")]
|
||
if len(parts) <= 1:
|
||
return f"FILE::{clean_relative}"
|
||
directory_parts = parts[:-1]
|
||
take = min(max(depth, 1), len(directory_parts))
|
||
return "/".join(directory_parts[:take])
|
||
|
||
|
||
def incremental_source_state(params: dict[str, str]) -> dict[str, Any]:
|
||
raw = params.get("SOURCE_STATE_JSON", "").strip()
|
||
if not raw:
|
||
return {"seenFingerprints": []}
|
||
try:
|
||
parsed = json.loads(raw)
|
||
except Exception as exc:
|
||
raise RuntimeError("SOURCE_STATE_JSON must be valid JSON") from exc
|
||
if not isinstance(parsed, dict):
|
||
raise RuntimeError("SOURCE_STATE_JSON must be a JSON object")
|
||
seen = parsed.get("seenFingerprints")
|
||
if seen is None:
|
||
parsed["seenFingerprints"] = []
|
||
elif not isinstance(seen, list):
|
||
raise RuntimeError("SOURCE_STATE_JSON.seenFingerprints must be an array")
|
||
else:
|
||
normalized_seen = []
|
||
for index, item in enumerate(seen):
|
||
if not isinstance(item, str):
|
||
raise RuntimeError(f"SOURCE_STATE_JSON.seenFingerprints[{index}] must be a string")
|
||
fingerprint = item.strip()
|
||
if fingerprint:
|
||
normalized_seen.append(fingerprint)
|
||
parsed["seenFingerprints"] = normalized_seen
|
||
incremental_unit_value = str(parsed.get("incrementalUnit", "")).strip().upper()
|
||
if incremental_unit_value and incremental_unit_value not in {"DIRECTORY", "OBJECT"}:
|
||
raise RuntimeError("SOURCE_STATE_JSON.incrementalUnit must be OBJECT or DIRECTORY")
|
||
parsed["incrementalUnit"] = incremental_unit_value or "OBJECT"
|
||
directory_depth = parsed.get("directoryDepth", 1)
|
||
if isinstance(directory_depth, bool) or not isinstance(directory_depth, int):
|
||
raise RuntimeError("SOURCE_STATE_JSON.directoryDepth must be an integer")
|
||
if directory_depth <= 0:
|
||
raise RuntimeError("SOURCE_STATE_JSON.directoryDepth must be greater than 0")
|
||
parsed["directoryDepth"] = directory_depth
|
||
return parsed
|
||
|
||
|
||
def apply_incremental_source_filter(
|
||
entries: list[dict[str, Any]],
|
||
data_source: str,
|
||
bucket: str,
|
||
params: dict[str, str],
|
||
source_path: str,
|
||
prefix: str,
|
||
pattern: str,
|
||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||
state = incremental_source_state(params)
|
||
seen = {str(item).strip() for item in state.get("seenFingerprints", []) if str(item).strip()}
|
||
scan_started_at = now_iso()
|
||
mode = incremental_unit(params)
|
||
depth = incremental_directory_depth(params)
|
||
emitted: list[dict[str, Any]] = []
|
||
updated_seen = set(seen)
|
||
if mode == "DIRECTORY":
|
||
grouped_entries: dict[str, list[dict[str, Any]]] = {}
|
||
for entry in entries:
|
||
group_key = directory_incremental_key(str(entry.get("path", "")).strip(), depth)
|
||
grouped_entries.setdefault(group_key, []).append(entry)
|
||
for group_key in sorted(grouped_entries.keys()):
|
||
group_entries = grouped_entries[group_key]
|
||
if group_key in updated_seen:
|
||
continue
|
||
updated_seen.add(group_key)
|
||
for entry in group_entries:
|
||
enriched = dict(entry)
|
||
enriched["fingerprint"] = source_entry_fingerprint(data_source, bucket, entry)
|
||
enriched["incrementalKey"] = group_key
|
||
emitted.append(enriched)
|
||
else:
|
||
for entry in entries:
|
||
fingerprint = source_entry_fingerprint(data_source, bucket, entry)
|
||
enriched = dict(entry)
|
||
enriched["fingerprint"] = fingerprint
|
||
enriched["incrementalKey"] = fingerprint
|
||
if fingerprint in updated_seen:
|
||
continue
|
||
updated_seen.add(fingerprint)
|
||
emitted.append(enriched)
|
||
scan_completed_at = now_iso()
|
||
checkpoint = {
|
||
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
|
||
"scanMode": "INCREMENTAL",
|
||
"incrementalUnit": mode,
|
||
"directoryDepth": depth,
|
||
"dataSource": data_source,
|
||
"sourcePath": source_path,
|
||
"bucket": bucket,
|
||
"prefix": prefix,
|
||
"pattern": pattern,
|
||
"seenFingerprints": sorted(updated_seen),
|
||
"listedCount": len(entries),
|
||
"newCount": len(emitted),
|
||
"scanStartedAt": scan_started_at,
|
||
"scanCompletedAt": scan_completed_at,
|
||
}
|
||
return emitted, checkpoint
|
||
|
||
|
||
def preview_payload_for_remote_entry(relative_path: str, mime_type: str, preview_url: str) -> dict[str, Any] | None:
|
||
normalized_preview_url = text_value(preview_url).strip()
|
||
if not normalized_preview_url:
|
||
return None
|
||
suffix = pathlib.PurePosixPath(relative_path).suffix.lower()
|
||
kind = "binary"
|
||
if suffix in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
|
||
kind = "video"
|
||
elif suffix in IMAGE_FILE_SUFFIXES or mime_type.startswith("image/"):
|
||
kind = "image"
|
||
elif suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
|
||
kind = "audio"
|
||
return {"preview": {"kind": kind, "format": suffix.lstrip(".") or kind, "url": normalized_preview_url}}
|
||
|
||
|
||
def source_entry_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
entry: dict[str, Any],
|
||
source_path: str,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
relative_path = str(entry.get("path", "")).strip().lstrip("/")
|
||
source = manifest_source_ref(entry, relative_path, source_path)
|
||
storage_path = str(source.get("path", "")).strip() or source_path_join(source_path, relative_path)
|
||
mime_type = str(entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
|
||
artifact_type = infer_port_type(pathlib.Path(relative_path))
|
||
payload: dict[str, Any] | None = None
|
||
if "data" in entry:
|
||
payload = normalize_json_payload(entry.get("data"))
|
||
else:
|
||
preview_payload = preview_payload_for_remote_entry(relative_path, mime_type, str(source.get("previewURL", "")).strip())
|
||
payload = preview_payload if preview_payload else None
|
||
if payload is not None:
|
||
payload["relativePath"] = relative_path
|
||
if source:
|
||
payload["source"] = source
|
||
metadata = clean_metadata({"mimeType": mime_type, "sha256": entry.get("sha256", "")})
|
||
return build_virtual_artifact(
|
||
run_id,
|
||
node_id,
|
||
pathlib.PurePosixPath(relative_path).name or "artifact",
|
||
storage_path,
|
||
artifact_type,
|
||
int(entry.get("sizeBytes", 0) or 0),
|
||
data=payload,
|
||
metadata=metadata,
|
||
output=output,
|
||
)
|
||
|
||
|
||
def collect_output_artifacts(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any] | None = None,
|
||
output: dict[str, Any] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
artifacts: list[dict[str, Any]] = []
|
||
for path in sorted(output_dir.rglob("*")):
|
||
if path.is_dir():
|
||
continue
|
||
artifact_type = infer_port_type(path)
|
||
mime_type, _ = mimetypes.guess_type(path.name)
|
||
if artifact_type == "json":
|
||
payload = load_payload_from_file(path)
|
||
else:
|
||
payload = extract_preview_data(path, artifact_type, mime_type or "")
|
||
binding = output
|
||
if binding is None and node_data is not None:
|
||
binding = output_binding(node_data)
|
||
artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding))
|
||
return artifacts
|
||
|
||
|
||
def hyperparams(node_data: dict[str, Any]) -> dict[str, str]:
|
||
values: dict[str, str] = {}
|
||
for item in node_data.get("hyperparams", []) or []:
|
||
key = str(item.get("key", "")).strip()
|
||
if not key:
|
||
continue
|
||
values[key] = str(item.get("value", ""))
|
||
return values
|
||
|
||
|
||
def parse_json_param(params: dict[str, str], key: str, default: Any, expected_type: type | tuple[type, ...] | None = None) -> Any:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
try:
|
||
parsed = json.loads(raw)
|
||
except Exception as exc:
|
||
raise RuntimeError(f"{key} must be valid JSON") from exc
|
||
if expected_type is not None and not isinstance(parsed, expected_type):
|
||
type_name = "array" if expected_type is list else "object" if expected_type is dict else "expected type"
|
||
raise RuntimeError(f"{key} must be a JSON {type_name}")
|
||
return parsed
|
||
|
||
|
||
def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
return [load_artifact_payload(item) for item in inputs]
|
||
|
||
|
||
def collect_items(payloads: list[Any]) -> list[Any]:
|
||
items: list[Any] = []
|
||
for payload in payloads:
|
||
payload = normalize_json_payload(payload)
|
||
if isinstance(payload.get("items"), list):
|
||
items.extend(payload["items"])
|
||
elif isinstance(payload.get("entries"), list):
|
||
items.extend(payload["entries"])
|
||
elif isinstance(payload.get("rows"), list):
|
||
items.extend(payload["rows"])
|
||
else:
|
||
items.append(payload)
|
||
return items
|
||
|
||
|
||
def normalize_mongo_document(item: Any) -> dict[str, Any]:
|
||
if isinstance(item, dict):
|
||
if set(item.keys()) == {"value"} and isinstance(item.get("value"), dict):
|
||
return json_compatible(item["value"])
|
||
return json_compatible(item)
|
||
return {"value": json_compatible(item)}
|
||
|
||
|
||
def documents_from_inputs(payloads: list[dict[str, Any]], document_path: str) -> list[dict[str, Any]]:
|
||
raw_items: list[Any] = []
|
||
path = str(document_path or "").strip()
|
||
if path:
|
||
for payload in payloads:
|
||
value = get_field_value(payload, path)
|
||
if isinstance(value, list):
|
||
raw_items.extend(value)
|
||
elif value is not None:
|
||
raw_items.append(value)
|
||
else:
|
||
raw_items = collect_items(payloads)
|
||
return [normalize_mongo_document(item) for item in raw_items]
|
||
|
||
|
||
def build_mongo_client(uri: str):
|
||
try:
|
||
from pymongo import MongoClient
|
||
except Exception as exc:
|
||
raise RuntimeError("pymongo is required for MongoDB 写入算子") from exc
|
||
return MongoClient(uri, serverSelectionTimeoutMS=5000)
|
||
|
||
|
||
def get_field_value(value: Any, field_path: str) -> Any:
|
||
current = value
|
||
for token in [part for part in field_path.split(".") if part]:
|
||
if isinstance(current, dict):
|
||
current = current.get(token)
|
||
continue
|
||
if isinstance(current, list) and token.isdigit():
|
||
index = int(token)
|
||
if 0 <= index < len(current):
|
||
current = current[index]
|
||
continue
|
||
return None
|
||
return current
|
||
|
||
|
||
def normalize_comparison_operator(value: str, label: str = "OPERATOR", default: str = "eq") -> str:
|
||
operator = str(value or "").strip().lower() or default
|
||
if operator not in SUPPORTED_COMPARISON_OPERATORS:
|
||
supported = ", ".join(sorted(SUPPORTED_COMPARISON_OPERATORS))
|
||
raise RuntimeError(f"{label} must be one of {supported}")
|
||
return operator
|
||
|
||
|
||
def normalize_sampling_rules(rules: list[Any], label: str = "RULES_JSON") -> list[dict[str, Any]]:
|
||
normalized_rules: list[dict[str, Any]] = []
|
||
rule_ids: set[str] = set()
|
||
for index, rule in enumerate(rules):
|
||
if not isinstance(rule, dict):
|
||
raise RuntimeError(f"{label}[{index}] must be an object")
|
||
for key in ("id", "field", "operator", "samplePercent"):
|
||
if key not in rule:
|
||
raise RuntimeError(f"{label}[{index}].{key} is required")
|
||
rule_id = str(rule.get("id", "")).strip()
|
||
field_path = str(rule.get("field", "")).strip()
|
||
if not rule_id:
|
||
raise RuntimeError(f"{label}[{index}].id must be non-empty")
|
||
if rule_id in rule_ids:
|
||
raise RuntimeError(f"{label} contains duplicate id: {rule_id}")
|
||
rule_ids.add(rule_id)
|
||
if not field_path:
|
||
raise RuntimeError(f"{label}[{index}].field must be non-empty")
|
||
operator = normalize_comparison_operator(str(rule.get("operator", "")), f"{label}[{index}].operator")
|
||
if operator not in {"exists", "missing"} and "value" not in rule:
|
||
raise RuntimeError(f"{label}[{index}].value is required")
|
||
sample_percent = parse_sample_percent(rule.get("samplePercent"), f"{label}[{index}].samplePercent")
|
||
normalized_rules.append(
|
||
{
|
||
"id": rule_id,
|
||
"field": field_path,
|
||
"operator": operator,
|
||
"value": rule.get("value"),
|
||
"samplePercent": sample_percent,
|
||
}
|
||
)
|
||
return normalized_rules
|
||
|
||
|
||
def parse_sample_percent(value: Any, label: str) -> float:
|
||
if isinstance(value, bool):
|
||
raise RuntimeError(f"{label} must be numeric")
|
||
try:
|
||
sample_percent = float(value)
|
||
except Exception as exc:
|
||
raise RuntimeError(f"{label} must be numeric") from exc
|
||
if not math.isfinite(sample_percent):
|
||
raise RuntimeError(f"{label} must be finite")
|
||
if sample_percent <= 0 or sample_percent > 100:
|
||
raise RuntimeError(f"{label} must be in the range (0, 100]")
|
||
return sample_percent
|
||
|
||
|
||
def normalize_field_map(field_map: dict[Any, Any], label: str = "FIELD_MAP_JSON") -> dict[str, str]:
|
||
normalized: dict[str, str] = {}
|
||
destinations: set[str] = set()
|
||
for source, destination in field_map.items():
|
||
if not isinstance(source, str):
|
||
raise RuntimeError(f"{label} source field names must be strings")
|
||
source_name = source.strip()
|
||
if not source_name:
|
||
raise RuntimeError(f"{label} source field names must be non-empty")
|
||
if source_name != source:
|
||
raise RuntimeError(f"{label} source field {source!r} must not have leading or trailing whitespace")
|
||
if not isinstance(destination, str):
|
||
raise RuntimeError(f"{label}.{source_name} destination field must be a string")
|
||
destination_name = destination.strip()
|
||
if not destination_name:
|
||
raise RuntimeError(f"{label}.{source_name} destination field must be non-empty")
|
||
if destination_name != destination:
|
||
raise RuntimeError(f"{label}.{source_name} destination field must not have leading or trailing whitespace")
|
||
if destination_name in destinations:
|
||
raise RuntimeError(f"{label} maps multiple source fields to destination field {destination_name!r}")
|
||
destinations.add(destination_name)
|
||
normalized[source_name] = destination_name
|
||
return normalized
|
||
|
||
|
||
def map_json_row(row: dict[str, Any], field_map: dict[str, str], label: str = "FIELD_MAP_JSON") -> dict[str, Any]:
|
||
mapped: dict[str, Any] = {}
|
||
mapped_sources: dict[str, str] = {}
|
||
for key, value in row.items():
|
||
output_key = field_map.get(key, key)
|
||
if output_key in mapped:
|
||
previous_source = mapped_sources[output_key]
|
||
raise RuntimeError(f"{label} maps source fields {previous_source!r} and {key!r} to the same output field {output_key!r}")
|
||
mapped[output_key] = value
|
||
mapped_sources[output_key] = key
|
||
return mapped
|
||
|
||
|
||
def compare_values(actual: Any, operator: str, expected: Any) -> bool:
|
||
if operator == "exists":
|
||
return actual is not None
|
||
if operator == "missing":
|
||
return actual is None
|
||
if operator == "contains":
|
||
if isinstance(actual, list):
|
||
return expected in actual
|
||
return str(expected) in str(actual)
|
||
if operator == "in":
|
||
if isinstance(expected, list):
|
||
return actual in expected
|
||
return str(actual) in str(expected)
|
||
if operator in {"gt", "gte", "lt", "lte"}:
|
||
try:
|
||
actual_number = float(actual)
|
||
expected_number = float(expected)
|
||
except Exception:
|
||
return False
|
||
if operator == "gt":
|
||
return actual_number > expected_number
|
||
if operator == "gte":
|
||
return actual_number >= expected_number
|
||
if operator == "lt":
|
||
return actual_number < expected_number
|
||
return actual_number <= expected_number
|
||
if operator == "ne":
|
||
return actual != expected
|
||
return actual == expected
|
||
|
||
|
||
def flatten_context(value: Any, prefix: str = "") -> dict[str, str]:
|
||
context: dict[str, str] = {}
|
||
if isinstance(value, dict):
|
||
for key, item in value.items():
|
||
next_prefix = f"{prefix}.{key}" if prefix else str(key)
|
||
context.update(flatten_context(item, next_prefix))
|
||
if prefix:
|
||
context[prefix] = json.dumps(value, ensure_ascii=False)
|
||
return context
|
||
if isinstance(value, list):
|
||
for index, item in enumerate(value[:8]):
|
||
next_prefix = f"{prefix}.{index}" if prefix else str(index)
|
||
context.update(flatten_context(item, next_prefix))
|
||
if prefix:
|
||
context[prefix] = json.dumps(value, ensure_ascii=False)
|
||
return context
|
||
if prefix:
|
||
context[prefix] = "" if value is None else str(value)
|
||
return context
|
||
|
||
|
||
def render_template_text(template_text: str, context: dict[str, str]) -> str:
|
||
rendered = template_text
|
||
for key, value in context.items():
|
||
rendered = rendered.replace("{{" + key + "}}", value)
|
||
safe_context = {key: value for key, value in context.items()}
|
||
|
||
class SafeDict(dict):
|
||
def __missing__(self, key: str) -> str:
|
||
return "{" + key + "}"
|
||
|
||
try:
|
||
rendered = rendered.format_map(SafeDict(safe_context))
|
||
except Exception:
|
||
pass
|
||
return rendered
|
||
|
||
|
||
def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
for item in collect_items(payloads):
|
||
if isinstance(item, dict):
|
||
rows.append(item)
|
||
else:
|
||
rows.append({"value": item})
|
||
return rows
|
||
|
||
|
||
def write_text_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
filename: str,
|
||
content: str,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
path = output_dir / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(content, encoding="utf-8")
|
||
mime_type, _ = mimetypes.guess_type(path.name)
|
||
preview = extract_preview_data(path, infer_port_type(path), mime_type or "")
|
||
return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output)
|
||
|
||
|
||
def write_csv_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
filename: str,
|
||
rows: list[dict[str, Any]],
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
path = output_dir / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
columns: list[str] = []
|
||
for row in rows:
|
||
for key in row.keys():
|
||
if key not in columns:
|
||
columns.append(key)
|
||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(handle, fieldnames=columns or ["value"])
|
||
writer.writeheader()
|
||
if rows:
|
||
writer.writerows(rows)
|
||
preview = preview_tabular_payload(path, ",", "csv")
|
||
return build_artifact(run_id, node_id, path, "file", data=preview, output=output)
|
||
|
||
|
||
def write_video_segment(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
|
||
cv2, _ = require_cv2()
|
||
capture = cv2.VideoCapture(str(source_path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频文件: {source_path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
start_frame = max(0, int(start_second * fps))
|
||
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
|
||
if frame_count > 0:
|
||
end_frame = min(end_frame, frame_count)
|
||
capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
|
||
if not writer.isOpened():
|
||
raise RuntimeError(f"无法创建视频切片: {destination}")
|
||
written = 0
|
||
try:
|
||
for _ in range(start_frame, end_frame):
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
break
|
||
writer.write(frame)
|
||
written += 1
|
||
finally:
|
||
writer.release()
|
||
duration = written / fps if fps > 0 else 0.0
|
||
return {
|
||
"frameRate": round(fps, 3),
|
||
"frameCount": written,
|
||
"resolution": f"{width}x{height}",
|
||
"duration": round(duration, 3),
|
||
}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def ffmpeg_available() -> bool:
|
||
return shutil.which("ffmpeg") is not None
|
||
|
||
|
||
def run_ffmpeg(args: list[str]) -> None:
|
||
completed = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||
if completed.returncode == 0:
|
||
return
|
||
stderr = (completed.stderr or "").strip()
|
||
stdout = (completed.stdout or "").strip()
|
||
message = stderr or stdout or "unknown ffmpeg error"
|
||
raise RuntimeError(f"ffmpeg execution failed: {message}")
|
||
|
||
|
||
def write_video_segment_with_ffmpeg(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
|
||
if not ffmpeg_available():
|
||
raise RuntimeError("ffmpeg is not available in operator-runner image")
|
||
duration = max(float(end_second) - float(start_second), 0.001)
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
copy_args = [
|
||
"ffmpeg",
|
||
"-nostdin",
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-y",
|
||
"-ss",
|
||
f"{max(float(start_second), 0.0):.3f}",
|
||
"-i",
|
||
str(source_path),
|
||
"-t",
|
||
f"{duration:.3f}",
|
||
"-map",
|
||
"0:v:0",
|
||
"-map",
|
||
"0:a?",
|
||
"-c",
|
||
"copy",
|
||
str(destination),
|
||
]
|
||
reencode_args = [
|
||
"ffmpeg",
|
||
"-nostdin",
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-y",
|
||
"-ss",
|
||
f"{max(float(start_second), 0.0):.3f}",
|
||
"-i",
|
||
str(source_path),
|
||
"-t",
|
||
f"{duration:.3f}",
|
||
"-map",
|
||
"0:v:0",
|
||
"-map",
|
||
"0:a?",
|
||
"-c:v",
|
||
"libx264",
|
||
"-preset",
|
||
"veryfast",
|
||
"-crf",
|
||
"23",
|
||
"-pix_fmt",
|
||
"yuv420p",
|
||
"-c:a",
|
||
"aac",
|
||
str(destination),
|
||
]
|
||
try:
|
||
run_ffmpeg(copy_args)
|
||
except RuntimeError:
|
||
run_ffmpeg(reencode_args)
|
||
metrics = inspect_video_file(destination)
|
||
return {
|
||
"frameRate": metrics.get("frameRate"),
|
||
"frameCount": metrics.get("frameCount"),
|
||
"resolution": metrics.get("resolution"),
|
||
"duration": metrics.get("duration", round(duration, 3)),
|
||
}
|
||
|
||
|
||
def write_video_segments(
|
||
source_path: pathlib.Path | str,
|
||
segments: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
cv2, _ = require_cv2()
|
||
capture = cv2.VideoCapture(str(source_path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频文件: {source_path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
normalized_segments: list[dict[str, Any]] = []
|
||
for raw in segments:
|
||
destination = pathlib.Path(raw.get("destination", ""))
|
||
start_second = float(raw.get("startSecond") or 0.0)
|
||
end_second = float(raw.get("endSecond") or 0.0)
|
||
start_frame = max(0, int(start_second * fps))
|
||
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
|
||
if frame_count > 0:
|
||
end_frame = min(end_frame, frame_count)
|
||
normalized_segments.append(
|
||
{
|
||
**raw,
|
||
"destination": destination,
|
||
"startFrame": start_frame,
|
||
"endFrame": end_frame,
|
||
}
|
||
)
|
||
normalized_segments.sort(key=lambda item: (int(item["startFrame"]), str(item["destination"])))
|
||
current_frame = 0
|
||
if normalized_segments:
|
||
initial_frame = int(normalized_segments[0]["startFrame"])
|
||
if initial_frame > 0:
|
||
capture.set(cv2.CAP_PROP_POS_FRAMES, initial_frame)
|
||
current_frame = initial_frame
|
||
results: list[dict[str, Any]] = []
|
||
for segment in normalized_segments:
|
||
start_frame = int(segment["startFrame"])
|
||
end_frame = int(segment["endFrame"])
|
||
destination = pathlib.Path(segment["destination"])
|
||
if current_frame < start_frame:
|
||
while current_frame < start_frame:
|
||
ok, _ = capture.read()
|
||
if not ok:
|
||
current_frame = end_frame
|
||
break
|
||
current_frame += 1
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
|
||
if not writer.isOpened():
|
||
raise RuntimeError(f"无法创建视频切片: {destination}")
|
||
written = 0
|
||
try:
|
||
while current_frame < end_frame:
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
break
|
||
writer.write(frame)
|
||
written += 1
|
||
current_frame += 1
|
||
finally:
|
||
writer.release()
|
||
results.append(
|
||
{
|
||
**segment,
|
||
"frameRate": round(fps, 3),
|
||
"frameCount": written,
|
||
"resolution": f"{width}x{height}",
|
||
"duration": round(written / fps, 3) if fps > 0 else 0.0,
|
||
}
|
||
)
|
||
return results
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def write_video_segment_from_remote_url(source_url: str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
|
||
return write_video_segment(source_url, destination, start_second, end_second)
|
||
|
||
|
||
def annotate_video_clip(path: pathlib.Path, frame_interval_seconds: float) -> dict[str, Any]:
|
||
cv2, np = require_cv2()
|
||
capture = cv2.VideoCapture(str(path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频切片: {path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
|
||
step = max(1, int(round(max(frame_interval_seconds, 0.1) * fps)))
|
||
previous_gray = None
|
||
samples: list[tuple[float, float]] = []
|
||
objects: set[str] = set()
|
||
frame_index = 0
|
||
while True:
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
break
|
||
if frame_index % step != 0:
|
||
frame_index += 1
|
||
continue
|
||
timestamp = frame_index / fps if fps > 0 else 0.0
|
||
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||
motion = 0.0
|
||
if previous_gray is not None:
|
||
diff = cv2.absdiff(previous_gray, gray)
|
||
motion = float(diff.mean() / 255.0)
|
||
previous_gray = gray
|
||
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
|
||
mask = cv2.inRange(ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8))
|
||
if float(mask.mean() / 255.0) > 0.1:
|
||
objects.add("手部")
|
||
_, threshold = cv2.threshold(gray, 24, 255, cv2.THRESH_BINARY)
|
||
contours, _ = cv2.findContours(threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||
if any(cv2.contourArea(contour) >= 64 for contour in contours):
|
||
objects.add("前景目标")
|
||
samples.append((timestamp, motion))
|
||
frame_index += 1
|
||
if not samples:
|
||
samples.append((0.0, 0.0))
|
||
|
||
def label_for_motion(motion: float) -> tuple[str, float]:
|
||
if motion < 0.015:
|
||
return "静止观察", 0.88
|
||
if motion < 0.045:
|
||
return "平稳活动", 0.79
|
||
return "快速移动", 0.71
|
||
|
||
actions: list[dict[str, Any]] = []
|
||
last_timestamp = duration
|
||
for timestamp, motion in samples:
|
||
label, confidence = label_for_motion(motion)
|
||
last_timestamp = timestamp
|
||
if not actions:
|
||
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
|
||
continue
|
||
current = actions[-1]
|
||
if current["label"] != label:
|
||
current["end"] = round(timestamp, 3)
|
||
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
|
||
else:
|
||
current["end"] = round(timestamp, 3)
|
||
current["confidence"] = round(min(float(current["confidence"]), confidence), 3)
|
||
if actions:
|
||
actions[-1]["end"] = round(max(duration, last_timestamp), 3)
|
||
min_confidence = min(float(item["confidence"]) for item in actions) if actions else 0.0
|
||
return {
|
||
"duration": round(duration, 3),
|
||
"actions": actions,
|
||
"objects": sorted(objects),
|
||
"minConfidence": round(min_confidence, 3),
|
||
"frameIntervalSeconds": frame_interval_seconds,
|
||
}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def builtin_source(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT"))
|
||
data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper()
|
||
if data_source and data_source not in VALID_SOURCE_DATA_SOURCES:
|
||
raise RuntimeError("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
|
||
source_path = (params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "")).strip()
|
||
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
|
||
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
|
||
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
|
||
bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
scan_mode = (params.get("SCAN_MODE") or "").strip().upper()
|
||
if scan_mode and scan_mode not in {"FULL", "INCREMENTAL"}:
|
||
raise RuntimeError("SCAN_MODE must be FULL or INCREMENTAL")
|
||
unit = incremental_unit(params)
|
||
depth = incremental_directory_depth(params)
|
||
source_bucket, source_prefix = parse_bos_path(source_path)
|
||
if not bucket and source_bucket:
|
||
bucket = source_bucket
|
||
if not prefix and source_prefix:
|
||
prefix = source_prefix
|
||
local_source_path = pathlib.Path(source_path) if source_path and not source_bucket else None
|
||
local_prefix_path = pathlib.Path(prefix) if prefix and "://" not in prefix else None
|
||
has_bos_connection = bool(bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key)
|
||
|
||
if inline_source:
|
||
data_source = "INLINE"
|
||
elif not data_source:
|
||
if source_bucket or has_bos_connection:
|
||
data_source = "BOS"
|
||
elif local_source_path is not None:
|
||
data_source = "LOCAL"
|
||
elif local_prefix_path is not None and local_prefix_path.exists():
|
||
data_source = "LOCAL"
|
||
elif inputs:
|
||
data_source = "UPSTREAM"
|
||
|
||
if data_source == "INLINE" and not inline_source:
|
||
raise RuntimeError("真实数据源 DATA_SOURCE=INLINE requires SOURCE_TEXT or SOURCE_JSON")
|
||
if data_source == "BOS":
|
||
missing = []
|
||
if not bucket:
|
||
missing.append("bucket or bos:// SOURCE_PATH")
|
||
if not bos_endpoint:
|
||
missing.append("BOS_ENDPOINT")
|
||
if not bos_access_key_id:
|
||
missing.append("BOS_ACCESS_KEY_ID")
|
||
if not bos_secret_access_key:
|
||
missing.append("BOS_SECRET_ACCESS_KEY")
|
||
if missing:
|
||
raise RuntimeError("真实数据源 DATA_SOURCE=BOS requires " + ", ".join(missing))
|
||
if data_source == "LOCAL":
|
||
if source_bucket:
|
||
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL cannot read a bos:// SOURCE_PATH")
|
||
if local_source_path is not None and not local_source_path.exists():
|
||
raise RuntimeError(f"真实数据源 SOURCE_PATH does not exist inside operator-runner: {source_path}")
|
||
if local_source_path is None and (local_prefix_path is None or not local_prefix_path.exists()):
|
||
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL requires SOURCE_PATH or an existing STORAGE_PREFIX path")
|
||
if not data_source:
|
||
raise RuntimeError(
|
||
"真实数据源 requires SOURCE_TEXT, SOURCE_JSON, SOURCE_PATH, a configured BOS source, "
|
||
"an existing local STORAGE_PREFIX path, or upstream input artifacts"
|
||
)
|
||
|
||
entries: list[dict[str, Any]] = []
|
||
resolved_source_path = source_path
|
||
if params.get("SOURCE_JSON"):
|
||
parsed = json.loads(params["SOURCE_JSON"])
|
||
if isinstance(parsed, list):
|
||
entries = [{"value": item} for item in parsed]
|
||
else:
|
||
entries = [{"value": parsed}]
|
||
elif params.get("SOURCE_TEXT"):
|
||
entries = [{"text": params["SOURCE_TEXT"]}]
|
||
elif data_source == "BOS":
|
||
cleaned_prefix = bos_prefix(prefix)
|
||
entries = list_bos_entries(
|
||
bucket,
|
||
cleaned_prefix,
|
||
pattern,
|
||
bos_endpoint,
|
||
bos_access_key_id,
|
||
bos_secret_access_key,
|
||
)
|
||
resolved_source_path = canonical_bos_path(bucket, cleaned_prefix)
|
||
elif local_source_path is not None:
|
||
entries = list_source_entries(local_source_path, pattern)
|
||
elif local_prefix_path is not None and local_prefix_path.exists():
|
||
entries = list_source_entries(local_prefix_path, pattern)
|
||
elif inputs:
|
||
entries = [load_artifact_payload(item) for item in inputs]
|
||
|
||
listed_entries = list(entries)
|
||
checkpoint_payload = None
|
||
if scan_mode == "INCREMENTAL":
|
||
entries, checkpoint_payload = apply_incremental_source_filter(
|
||
listed_entries,
|
||
data_source,
|
||
bucket,
|
||
params,
|
||
resolved_source_path,
|
||
prefix,
|
||
pattern,
|
||
)
|
||
|
||
payload = {
|
||
"dataSource": data_source,
|
||
"bucket": bucket,
|
||
"prefix": prefix,
|
||
"endpoint": bos_endpoint,
|
||
"pattern": pattern,
|
||
"scanMode": scan_mode or "FULL",
|
||
"incrementalUnit": unit,
|
||
"directoryDepth": depth,
|
||
"sourcePath": resolved_source_path,
|
||
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
|
||
"listedCount": len(listed_entries),
|
||
"newCount": len(entries) if checkpoint_payload is not None else len(entries),
|
||
"entries": entries,
|
||
"catalogEntries": listed_entries,
|
||
}
|
||
artifacts = [
|
||
write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))
|
||
]
|
||
if checkpoint_payload is not None:
|
||
artifacts.append(write_json_artifact(run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload))
|
||
|
||
if data_source == "BOS":
|
||
for entry in entries:
|
||
artifacts.append(source_entry_artifact(run_id, node_id, entry, resolved_source_path, output=output_binding(node_data)))
|
||
elif source_path and pathlib.Path(source_path).exists():
|
||
source_root = pathlib.Path(source_path)
|
||
staged_root = output_dir / "_source_tree"
|
||
for entry in entries:
|
||
relative_path = str(entry.get("path", "")).strip()
|
||
if not relative_path:
|
||
continue
|
||
candidate = source_root / relative_path if source_root.is_dir() else source_root
|
||
if not candidate.exists() or not candidate.is_file():
|
||
continue
|
||
destination = staged_root / relative_path
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(candidate, destination)
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, staged_root, node_data))
|
||
|
||
return artifacts
|
||
|
||
|
||
def builtin_merge(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = payloads_from_inputs(inputs)
|
||
merged = merge_dicts(payloads)
|
||
merged["inputCount"] = len(payloads)
|
||
return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_intersection(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = payloads_from_inputs(inputs)
|
||
if not payloads:
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"intersection.json",
|
||
{"keys": [], "inputCount": 0},
|
||
output=output_binding(node_data),
|
||
)
|
||
]
|
||
key_sets = [set(payload.keys()) for payload in payloads]
|
||
common_keys = sorted(set.intersection(*key_sets)) if key_sets else []
|
||
payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_distribution(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = payloads_from_inputs(inputs)
|
||
items = collect_items(payloads)
|
||
outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}]
|
||
partitions: list[list[Any]] = [[] for _ in outputs]
|
||
for index, item in enumerate(items):
|
||
partitions[index % len(outputs)].append(item)
|
||
artifacts: list[dict[str, Any]] = []
|
||
for index, output in enumerate(outputs):
|
||
filename = f"{output.get('name', f'queue-{index}')}.json"
|
||
payload = {"items": partitions[index], "count": len(partitions[index])}
|
||
artifacts.append(
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
filename,
|
||
payload,
|
||
str(output.get("type", "json")),
|
||
output=output,
|
||
)
|
||
)
|
||
return artifacts
|
||
|
||
|
||
def builtin_upload(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip()
|
||
bucket = str(node_data.get("bucket", "")).strip()
|
||
prefix = str(node_data.get("prefix", "")).strip()
|
||
parsed_bucket, parsed_prefix = parse_bos_path(upload_path)
|
||
if parsed_bucket:
|
||
bucket = parsed_bucket
|
||
if parsed_prefix or upload_path.endswith("/"):
|
||
prefix = parsed_prefix
|
||
data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper()
|
||
bos_endpoint = params.get("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
copied_paths: list[str] = []
|
||
bundle_inputs: list[dict[str, Any]] = []
|
||
staging_dir = output_dir / "uploaded"
|
||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||
package_manifest_payload = first_payload_with_keys(inputs, "packageCount", "items")
|
||
upload_inputs = list(inputs)
|
||
if package_manifest_payload is not None:
|
||
package_names = package_related_artifact_names(package_manifest_payload)
|
||
upload_inputs = [
|
||
artifact
|
||
for artifact in inputs
|
||
if str(artifact.get("name", "")).strip() not in package_names
|
||
]
|
||
upload_inputs.extend(
|
||
materialize_package_artifacts_from_manifest(
|
||
run_id,
|
||
node_id,
|
||
staging_dir,
|
||
package_manifest_payload,
|
||
node_data,
|
||
)
|
||
)
|
||
|
||
if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key:
|
||
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
||
temp_paths = [
|
||
str(item.get("path", "")).strip()
|
||
for item in upload_inputs
|
||
if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip()
|
||
]
|
||
input_root = common_input_root(temp_paths)
|
||
uploaded_objects: list[dict[str, Any]] = []
|
||
remote_artifacts: list[dict[str, Any]] = []
|
||
|
||
for index, artifact in enumerate(upload_inputs):
|
||
payload = load_artifact_payload(artifact)
|
||
bundle_inputs.append(payload)
|
||
fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}"
|
||
relative_path = upload_relative_path(artifact, input_root, fallback_name)
|
||
temp_source = materialized_upload_source(artifact, staging_dir, relative_path)
|
||
if temp_source is None or not temp_source.exists():
|
||
continue
|
||
|
||
mime_type = (
|
||
str((artifact.get("metadata") or {}).get("mimeType", "")).strip()
|
||
if isinstance(artifact.get("metadata"), dict)
|
||
else ""
|
||
) or mimetypes.guess_type(temp_source.name)[0] or ""
|
||
object_key = bos_key_join(prefix, relative_path)
|
||
put_kwargs = {"content_type": mime_type} if mime_type else {}
|
||
try:
|
||
size_bytes = temp_source.stat().st_size
|
||
except OSError:
|
||
size_bytes = 0
|
||
log(
|
||
f"[operator-runner][info] upload starts: {temp_source} -> {canonical_bos_path(bucket, object_key)} "
|
||
f"({size_bytes} bytes)"
|
||
)
|
||
client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs)
|
||
remote_path = canonical_bos_path(bucket, object_key)
|
||
log(
|
||
f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} "
|
||
f"({size_bytes} bytes)"
|
||
)
|
||
preview_url = ""
|
||
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
|
||
preview_url = text_value(
|
||
client.generate_pre_signed_url(
|
||
bucket,
|
||
object_key,
|
||
expiration_in_seconds=3600,
|
||
protocol=scheme,
|
||
)
|
||
).strip()
|
||
|
||
uploaded_objects.append(
|
||
clean_metadata(
|
||
{
|
||
"name": pathlib.PurePosixPath(relative_path).name,
|
||
"relativePath": relative_path,
|
||
"path": remote_path,
|
||
"previewURL": preview_url,
|
||
"sizeBytes": temp_source.stat().st_size,
|
||
"mimeType": mime_type,
|
||
}
|
||
)
|
||
or {}
|
||
)
|
||
|
||
artifact_type = str(artifact.get("type", "")).strip() or infer_port_type(temp_source)
|
||
artifact_payload: dict[str, Any] | None = None
|
||
if artifact_type == "json":
|
||
loaded = load_payload_from_file(temp_source)
|
||
if loaded is not None:
|
||
artifact_payload = normalize_json_payload(loaded)
|
||
else:
|
||
artifact_payload = preview_payload_for_remote_entry(relative_path, mime_type, preview_url)
|
||
if artifact_payload is None:
|
||
artifact_payload = extract_preview_data(temp_source, artifact_type, mime_type)
|
||
if artifact_payload is not None:
|
||
artifact_payload["relativePath"] = relative_path
|
||
artifact_payload["source"] = clean_metadata(
|
||
{
|
||
"path": remote_path,
|
||
"previewURL": preview_url,
|
||
"relativePath": relative_path,
|
||
"mimeType": mime_type,
|
||
}
|
||
) or {}
|
||
|
||
source_size_bytes = temp_source.stat().st_size
|
||
source_sha256, source_sha256_metadata = artifact_sha256(temp_source, source_size_bytes)
|
||
|
||
remote_artifacts.append(
|
||
build_virtual_artifact(
|
||
run_id,
|
||
node_id,
|
||
pathlib.PurePosixPath(relative_path).name,
|
||
remote_path,
|
||
artifact_type,
|
||
source_size_bytes,
|
||
data=artifact_payload,
|
||
metadata={
|
||
**source_sha256_metadata,
|
||
"sha256": source_sha256,
|
||
"mimeType": mime_type,
|
||
},
|
||
output=output_binding(node_data),
|
||
)
|
||
)
|
||
|
||
payload = {
|
||
"copiedPaths": copied_paths,
|
||
"uploadedCount": len(uploaded_objects),
|
||
"uploadedObjects": uploaded_objects,
|
||
"items": bundle_inputs,
|
||
"inputCount": len(inputs),
|
||
"dataSource": "BOS",
|
||
"bucket": bucket,
|
||
"prefix": prefix,
|
||
"destinationPath": canonical_bos_path(bucket, prefix),
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
|
||
artifacts.extend(remote_artifacts)
|
||
return artifacts
|
||
|
||
for artifact in upload_inputs:
|
||
payload = load_artifact_payload(artifact)
|
||
bundle_inputs.append(payload)
|
||
source_path = artifact.get("path")
|
||
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
|
||
source = pathlib.Path(source_path)
|
||
destination = staging_dir / source.name
|
||
if source.resolve() != destination.resolve():
|
||
shutil.copy2(source, destination)
|
||
copied_paths.append(str(destination))
|
||
|
||
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir))
|
||
return artifacts
|
||
|
||
|
||
def builtin_json_extract(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_path = params.get("FIELD_PATH", "").strip()
|
||
payloads = payloads_from_inputs(inputs)
|
||
payload = payloads[0] if payloads else {}
|
||
extracted = get_field_value(payload, field_path) if field_path else payload
|
||
result = {"fieldPath": field_path, "value": extracted}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_json_filter(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_path = params.get("FIELD_PATH", "").strip()
|
||
operator = normalize_comparison_operator(params.get("OPERATOR", "eq"))
|
||
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
|
||
rows = collect_items(payloads_from_inputs(inputs))
|
||
filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
|
||
payload = {
|
||
"fieldPath": field_path,
|
||
"operator": operator,
|
||
"expected": expected,
|
||
"items": filtered,
|
||
"count": len(filtered),
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_json_map(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_map = normalize_field_map(parse_json_param(params, "FIELD_MAP_JSON", {}, dict))
|
||
rows = rows_from_payloads(payloads_from_inputs(inputs))
|
||
mapped = [map_json_row(row, field_map) for row in rows]
|
||
payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_mongodb_write(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
mongo_target = params.get("MONGO_TARGET", "").strip().lower() or "business"
|
||
explicit_mongo_uri = params.get("MONGO_URI", "").strip() or os.getenv("MONGO_URI", "").strip()
|
||
explicit_mongo_db = params.get("MONGO_DB", "").strip() or os.getenv("MONGO_DB", "").strip()
|
||
collection_name = params.get("MONGO_COLLECTION", "").strip()
|
||
mode = params.get("MONGO_MODE", "insert_many").strip().lower() or "insert_many"
|
||
upsert_key = params.get("UPSERT_KEY", "_id").strip() or "_id"
|
||
document_path = params.get("DOCUMENT_PATH", "").strip()
|
||
|
||
mongo_uri = explicit_mongo_uri
|
||
mongo_db = explicit_mongo_db
|
||
if not mongo_uri:
|
||
if mongo_target == "platform":
|
||
mongo_uri = os.getenv("PLATFORM_MONGO_URI", "").strip()
|
||
mongo_db = mongo_db or os.getenv("PLATFORM_MONGO_DB", "").strip()
|
||
else:
|
||
mongo_uri = os.getenv("BUSINESS_MONGO_URI", "").strip()
|
||
mongo_db = mongo_db or os.getenv("BUSINESS_MONGO_DB", "").strip()
|
||
|
||
if not mongo_uri:
|
||
raise RuntimeError(f"MongoDB 写入缺少可用连接:target={mongo_target}")
|
||
if not mongo_db:
|
||
raise RuntimeError(f"MongoDB 写入缺少数据库名:target={mongo_target}")
|
||
if not collection_name:
|
||
raise RuntimeError("MONGO_COLLECTION is required for MongoDB 写入")
|
||
if mongo_target == "platform" and collection_name.lower() in {item.lower() for item in RESERVED_MONGO_COLLECTIONS}:
|
||
raise RuntimeError(f"collection {collection_name} is reserved and cannot be written by workflow operators")
|
||
if mode not in {"insert_many", "upsert"}:
|
||
raise RuntimeError("MONGO_MODE must be insert_many or upsert")
|
||
|
||
documents = documents_from_inputs(payloads_from_inputs(inputs), document_path)
|
||
if not documents:
|
||
raise RuntimeError("MongoDB 写入未收到可写入的文档")
|
||
|
||
client = build_mongo_client(mongo_uri)
|
||
try:
|
||
collection = client[mongo_db][collection_name]
|
||
summary: dict[str, Any] = {
|
||
"target": mongo_target,
|
||
"database": mongo_db,
|
||
"collection": collection_name,
|
||
"mode": mode,
|
||
"documentCount": len(documents),
|
||
}
|
||
if document_path:
|
||
summary["documentPath"] = document_path
|
||
if mode == "insert_many":
|
||
result = collection.insert_many(documents, ordered=False)
|
||
summary["insertedCount"] = len(getattr(result, "inserted_ids", []) or [])
|
||
else:
|
||
processed = 0
|
||
matched_count = 0
|
||
modified_count = 0
|
||
upserted_ids: list[Any] = []
|
||
for document in documents:
|
||
key_value = get_field_value(document, upsert_key)
|
||
if key_value is None:
|
||
raise RuntimeError(f"document is missing UPSERT_KEY={upsert_key}")
|
||
result = collection.replace_one({upsert_key: key_value}, document, upsert=True)
|
||
processed += 1
|
||
matched_count += int(getattr(result, "matched_count", 0) or 0)
|
||
modified_count += int(getattr(result, "modified_count", 0) or 0)
|
||
upserted_id = getattr(result, "upserted_id", None)
|
||
if upserted_id is not None:
|
||
upserted_ids.append(json_compatible(upserted_id))
|
||
summary["upsertKey"] = upsert_key
|
||
summary["upsertedCount"] = processed
|
||
summary["matchedCount"] = matched_count
|
||
summary["modifiedCount"] = modified_count
|
||
if upserted_ids:
|
||
summary["upsertedIds"] = upserted_ids
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"mongodb-write-result.json",
|
||
summary,
|
||
output=output_binding(node_data),
|
||
)
|
||
]
|
||
finally:
|
||
close = getattr(client, "close", None)
|
||
if callable(close):
|
||
close()
|
||
|
||
|
||
def builtin_text_template(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
template_text = params.get("TEMPLATE_TEXT", "").strip()
|
||
output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt"
|
||
payloads = payloads_from_inputs(inputs)
|
||
context = flatten_context(payloads[0] if payloads else {})
|
||
rendered = render_template_text(template_text, context)
|
||
binding = output_binding(node_data)
|
||
manifest = write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"rendered-template.json",
|
||
{"template": template_text, "context": context, "outputName": output_name},
|
||
output=binding,
|
||
)
|
||
return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)]
|
||
|
||
|
||
def builtin_csv_to_json(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
payload: dict[str, Any]
|
||
if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES:
|
||
delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()]
|
||
columns, rows = load_tabular_rows(source_path, delimiter)
|
||
payload = {
|
||
"sourcePath": str(source_path),
|
||
"columns": columns,
|
||
"items": rows,
|
||
"rowCount": len(rows),
|
||
}
|
||
else:
|
||
for artifact in inputs:
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if preview.get("kind") == "table":
|
||
payload = {
|
||
"sourcePath": str(artifact.get("path", "")),
|
||
"columns": preview.get("columns", []),
|
||
"items": preview.get("rows", []),
|
||
"rowCount": preview.get("rowCount", len(preview.get("rows", []))),
|
||
"previewTruncated": preview.get("truncated", False),
|
||
}
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"table.json",
|
||
payload,
|
||
output=output_binding(node_data),
|
||
)
|
||
]
|
||
raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件")
|
||
return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_json_to_csv(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
rows = rows_from_payloads(payloads_from_inputs(inputs))
|
||
binding = output_binding(node_data)
|
||
summary = write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"csv-export-summary.json",
|
||
{"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})},
|
||
output=binding,
|
||
)
|
||
return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)]
|
||
|
||
|
||
def builtin_directory_scan(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
root = source_path if source_path.is_dir() else source_path.parent
|
||
payload = {
|
||
"sourcePath": str(source_path),
|
||
"scanRoot": str(root),
|
||
"summary": summarize_directory_tree(root),
|
||
}
|
||
elif inputs:
|
||
payload = {
|
||
"sourcePath": "",
|
||
"scanRoot": "",
|
||
"summary": summarize_input_artifacts(inputs),
|
||
}
|
||
else:
|
||
raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录")
|
||
return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_xplatform_index(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payload = load_xplatform_index_payload(hyperparams(node_data), inputs, output_dir)
|
||
return [write_json_artifact(run_id, node_id, output_dir, "dataset-index.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_collection_validate(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
require_intrinsics = parse_bool_param(params, "REQUIRE_INTRINSICS", False)
|
||
payload = first_payload_with_keys(inputs, "collections", "items")
|
||
if payload is None:
|
||
payload = load_xplatform_index_payload(params, inputs, output_dir)
|
||
collections = []
|
||
valid_collection_count = 0
|
||
for collection in payload.get("collections", []):
|
||
if not isinstance(collection, dict):
|
||
continue
|
||
missing_required = list(collection.get("missingRequiredFiles", []))
|
||
if require_intrinsics and not collection.get("hasIntrinsics", False):
|
||
missing_required.append("intrinsics.json")
|
||
valid = len(missing_required) == 0 and int(collection.get("videoCount", 0) or 0) > 0
|
||
if valid:
|
||
valid_collection_count += 1
|
||
collections.append({**collection, "missingRequiredFiles": missing_required, "valid": valid})
|
||
items = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
collection_id = str(item.get("collectionId", "")).strip()
|
||
items.append({**item, "collectionValid": any(c.get("id") == collection_id and c.get("valid", False) for c in collections)})
|
||
result = {
|
||
**payload,
|
||
"datasetType": "x-platform-validation",
|
||
"collections": collections,
|
||
"items": items,
|
||
"validCollectionCount": valid_collection_count,
|
||
"invalidCollectionCount": max(len(collections) - valid_collection_count, 0),
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "dataset-validation.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_video_quality(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
min_clarity = parse_unit_interval_param(params, "MIN_CLARITY", DEFAULT_MIN_CLARITY)
|
||
max_shake = parse_unit_interval_param(params, "MAX_SHAKE", DEFAULT_MAX_SHAKE)
|
||
max_hand_occlusion = parse_unit_interval_param(params, "MAX_HAND_OCCLUSION", DEFAULT_MAX_HAND_OCCLUSION)
|
||
clarity_scale = parse_positive_float_param(params, "CLARITY_SCALE", 35.0)
|
||
payload = first_payload_with_keys(inputs, "items")
|
||
if payload is None:
|
||
payload = load_xplatform_index_payload(params, inputs, output_dir)
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
items = []
|
||
passed_count = 0
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
source_path = source_path_from_item(item, str(item.get("filename", "video.mp4")), output_dir, archive_sources)
|
||
metrics = analyze_video_quality_metrics(source_path, clarity_scale)
|
||
passed = (
|
||
float_value(metrics.get("clarity"), 0.0) >= min_clarity
|
||
and float_value(metrics.get("shake"), 1.0) <= max_shake
|
||
and float_value(metrics.get("handOcclusion"), 1.0) <= max_hand_occlusion
|
||
)
|
||
if passed:
|
||
passed_count += 1
|
||
items.append(
|
||
{
|
||
**item,
|
||
"quality": {
|
||
**metrics,
|
||
"passed": passed,
|
||
"thresholds": {
|
||
"minClarity": min_clarity,
|
||
"maxShake": max_shake,
|
||
"maxHandOcclusion": max_hand_occlusion,
|
||
},
|
||
},
|
||
}
|
||
)
|
||
result = {
|
||
**payload,
|
||
"datasetType": "x-platform-quality",
|
||
"items": items,
|
||
"videoCount": len(items),
|
||
"passedCount": passed_count,
|
||
"failedCount": max(len(items) - passed_count, 0),
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "quality-report.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_dataset_filter(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "items")
|
||
if payload is None:
|
||
raise RuntimeError("数据过滤需要上游数据清单输入")
|
||
date_start = normalize_date_value(params.get("DATE_START", ""))
|
||
date_end = normalize_date_value(params.get("DATE_END", ""))
|
||
place = params.get("PLACE", "").strip()
|
||
collection_type = params.get("TYPE", "").strip()
|
||
min_clarity = parse_unit_interval_param(params, "MIN_CLARITY", DEFAULT_MIN_CLARITY)
|
||
max_shake = parse_unit_interval_param(params, "MAX_SHAKE", DEFAULT_MAX_SHAKE)
|
||
max_hand_occlusion = parse_unit_interval_param(params, "MAX_HAND_OCCLUSION", DEFAULT_MAX_HAND_OCCLUSION)
|
||
filtered = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
meta = item.get("meta") if isinstance(item.get("meta"), dict) else {}
|
||
quality = item.get("quality") if isinstance(item.get("quality"), dict) else {}
|
||
if place and str(meta.get("place", "")).strip() != place:
|
||
continue
|
||
if collection_type and str(meta.get("type", "")).strip() != collection_type:
|
||
continue
|
||
if date_start or date_end:
|
||
if not date_in_range(meta.get("date", ""), date_start, date_end):
|
||
continue
|
||
if float_value(quality.get("clarity"), 0.0) < min_clarity:
|
||
continue
|
||
if float_value(quality.get("shake"), 1.0) > max_shake:
|
||
continue
|
||
if float_value(quality.get("handOcclusion"), 1.0) > max_hand_occlusion:
|
||
continue
|
||
if not bool(quality.get("passed", False)):
|
||
continue
|
||
filtered.append(item)
|
||
result = {
|
||
"datasetType": "x-platform-filtered",
|
||
"criteria": {
|
||
"dateStart": date_start,
|
||
"dateEnd": date_end,
|
||
"place": place,
|
||
"type": collection_type,
|
||
"minClarity": min_clarity,
|
||
"maxShake": max_shake,
|
||
"maxHandOcclusion": max_hand_occlusion,
|
||
},
|
||
"count": len(filtered),
|
||
"items": filtered,
|
||
}
|
||
if isinstance(payload.get("archiveSources"), dict):
|
||
result["archiveSources"] = payload["archiveSources"]
|
||
return [write_json_artifact(run_id, node_id, output_dir, "filtered-videos.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_video_slicing(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "items")
|
||
if payload is None:
|
||
raise RuntimeError("视频切片需要上游过滤结果输入")
|
||
segment_duration = parse_positive_int_param(params, "SEGMENT_DURATION_SECONDS", 300)
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
clips: list[dict[str, Any]] = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
source_filename = str(item.get("filename", "")).strip() or "video.mp4"
|
||
source_path = source_path_from_item(item, source_filename, output_dir, archive_sources)
|
||
metrics = inspect_video_file(source_path)
|
||
duration = float(item.get("duration") or metrics.get("duration") or 0.0)
|
||
if duration <= 0:
|
||
duration = float(metrics.get("duration") or 0.0)
|
||
if duration <= 0:
|
||
duration = float(segment_duration)
|
||
clip_count = max(1, int(math.ceil(duration / segment_duration)))
|
||
stem = pathlib.Path(source_filename).stem
|
||
clip_specs: list[dict[str, Any]] = []
|
||
for clip_index in range(clip_count):
|
||
clip_start = clip_index * segment_duration
|
||
clip_end = min(duration, clip_start + segment_duration)
|
||
clip_name = f"{stem}_clip{clip_index + 1:03d}.mp4"
|
||
clip_path = output_dir / "clips" / clip_name
|
||
clip_specs.append(
|
||
{
|
||
"clipIndex": clip_index,
|
||
"clipStart": clip_start,
|
||
"clipEnd": clip_end,
|
||
"clipName": clip_name,
|
||
"clipPath": clip_path,
|
||
"destination": clip_path,
|
||
"startSecond": clip_start,
|
||
"endSecond": clip_end,
|
||
}
|
||
)
|
||
clip_metrics_list: list[dict[str, Any]] | None = None
|
||
if ffmpeg_available():
|
||
try:
|
||
log(f"[operator-runner][info] video_slicing uses ffmpeg for {source_filename} ({clip_count} clip(s))")
|
||
clip_metrics_list = []
|
||
for clip_spec in clip_specs:
|
||
clip_metrics_list.append(
|
||
{
|
||
**clip_spec,
|
||
**write_video_segment_with_ffmpeg(
|
||
source_path,
|
||
pathlib.Path(clip_spec["clipPath"]),
|
||
float(clip_spec["clipStart"]),
|
||
float(clip_spec["clipEnd"]),
|
||
),
|
||
}
|
||
)
|
||
except RuntimeError as exc:
|
||
log(f"[operator-runner][warn] ffmpeg video slicing failed for {source_filename}, fallback to opencv: {exc}")
|
||
clip_metrics_list = None
|
||
if clip_metrics_list is None:
|
||
if len(clip_specs) > 1:
|
||
clip_metrics_list = write_video_segments(source_path, clip_specs)
|
||
else:
|
||
only = clip_specs[0]
|
||
clip_metrics_list = [
|
||
{
|
||
**only,
|
||
**write_video_segment(source_path, only["clipPath"], float(only["clipStart"]), float(only["clipEnd"])),
|
||
}
|
||
]
|
||
for clip_spec in clip_metrics_list:
|
||
clip_start = float(clip_spec["clipStart"])
|
||
clip_end = float(clip_spec["clipEnd"])
|
||
clip_name = str(clip_spec["clipName"])
|
||
clip_path = pathlib.Path(clip_spec["clipPath"])
|
||
clips.append(
|
||
{
|
||
"clipId": pathlib.Path(clip_name).stem,
|
||
"clipName": clip_name,
|
||
"clipPath": str(clip_path),
|
||
"sourceVideo": source_filename,
|
||
"sourcePath": str(source_path),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
"collectionId": item.get("collectionId", ""),
|
||
"meta": item.get("meta", {}),
|
||
"quality": item.get("quality", {}),
|
||
"startSecond": round(clip_start, 3),
|
||
"endSecond": round(clip_end, 3),
|
||
"duration": clip_spec.get("duration", round(max(clip_end - clip_start, 0.0), 3)),
|
||
"resolution": clip_spec.get("resolution"),
|
||
"frameRate": clip_spec.get("frameRate"),
|
||
}
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-clips",
|
||
"segmentDurationSeconds": segment_duration,
|
||
"clipCount": len(clips),
|
||
"items": clips,
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "clips-manifest.json", manifest, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "clips", node_data=node_data, output=output_binding(node_data)))
|
||
return artifacts
|
||
|
||
|
||
def builtin_video_annotation(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "clipCount", "items")
|
||
if payload is None:
|
||
raise RuntimeError("视频标注需要上游切片结果输入")
|
||
frame_interval = parse_positive_float_param(params, "FRAME_INTERVAL_SECONDS", 1.0)
|
||
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
items = []
|
||
for clip in payload.get("items", []):
|
||
if not isinstance(clip, dict):
|
||
continue
|
||
clip_name = str(clip.get("clipName", "")).strip() or pathlib.Path(str(clip.get("clipPath", ""))).name
|
||
artifact = clip_artifacts.get(clip_name)
|
||
clip_path: pathlib.Path | None = None
|
||
if artifact is not None:
|
||
try:
|
||
clip_path = source_path_from_item(
|
||
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
|
||
clip_name,
|
||
output_dir,
|
||
archive_sources,
|
||
)
|
||
except RuntimeError:
|
||
clip_path = None
|
||
if clip_path is None:
|
||
direct_clip_path = str(clip.get("clipPath", "")).strip()
|
||
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
|
||
clip_path = pathlib.Path(direct_clip_path)
|
||
if clip_path is None:
|
||
clip_path = materialize_clip_from_source(clip, output_dir, archive_sources)
|
||
if clip_path is None or not clip_path.exists():
|
||
raise RuntimeError(f"无法获取真实输入文件: {clip_name}")
|
||
annotation = annotate_video_clip(clip_path, frame_interval)
|
||
annotation_filename = f"{pathlib.Path(clip_name).stem}_annotation.json"
|
||
annotation_payload = {
|
||
"clipId": clip.get("clipId", pathlib.Path(clip_name).stem),
|
||
"sourceVideo": clip.get("sourceVideo", ""),
|
||
**annotation,
|
||
}
|
||
items.append({**clip, "annotation": annotation_payload, "annotationFile": annotation_filename})
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir / "annotations",
|
||
annotation_filename,
|
||
annotation_payload,
|
||
output=output_binding(node_data),
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-annotations",
|
||
"annotationCount": len(items),
|
||
"items": items,
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "annotations-manifest.json", manifest, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "annotations", node_data=node_data, output=output_binding(node_data)))
|
||
return artifacts
|
||
|
||
|
||
def builtin_result_package(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payload = first_payload_with_keys(inputs, "annotationCount", "items")
|
||
if payload is None:
|
||
payload = first_payload_with_keys(inputs, "clipCount", "items")
|
||
if payload is None:
|
||
raise RuntimeError("成品打包需要上游标注或目标裁剪结果输入")
|
||
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
packages = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
clip_name = str(item.get("clipName", "")).strip()
|
||
clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem
|
||
package_name = f"{clip_id}_package"
|
||
package_dir = output_dir / "packages" / package_name
|
||
package_dir.mkdir(parents=True, exist_ok=True)
|
||
artifact = clip_artifacts.get(clip_name)
|
||
clip_path: pathlib.Path | None = None
|
||
if artifact is not None:
|
||
try:
|
||
clip_path = source_path_from_item(
|
||
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
|
||
clip_name,
|
||
output_dir,
|
||
archive_sources,
|
||
)
|
||
except RuntimeError:
|
||
clip_path = None
|
||
if clip_path is None:
|
||
direct_clip_path = str(item.get("clipPath", "")).strip()
|
||
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
|
||
clip_path = pathlib.Path(direct_clip_path)
|
||
if clip_path is None:
|
||
clip_path = materialize_clip_from_source(item, output_dir, archive_sources)
|
||
if clip_path is not None and clip_path.exists():
|
||
shutil.copy2(clip_path, package_dir / clip_name)
|
||
quality_payload = {
|
||
"clip_id": clip_id,
|
||
"source_video": item.get("sourceVideo", ""),
|
||
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
|
||
}
|
||
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
|
||
if not annotation_payload and isinstance(item.get("detections"), list):
|
||
annotation_payload = {
|
||
"type": "target_detections",
|
||
"detectionMode": item.get("detectionMode", payload.get("detectionMode", "")),
|
||
"targetLabels": item.get("targetLabels", payload.get("targetLabels", [])),
|
||
"detections": item.get("detections", []),
|
||
"detectionCount": item.get("detectionCount", 0),
|
||
"maxConfidence": item.get("maxConfidence", 0),
|
||
}
|
||
config_payload = package_config_payload(item, payload)
|
||
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_quality.json", quality_payload, output=output_binding(node_data))
|
||
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation_payload, output=output_binding(node_data))
|
||
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_config.json", config_payload, output=output_binding(node_data))
|
||
package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()]
|
||
packages.append(
|
||
{
|
||
"packageName": package_name,
|
||
"clipId": clip_id,
|
||
"clipName": clip_name,
|
||
"sourceVideo": item.get("sourceVideo", ""),
|
||
"clipDuration": item.get("duration", 0),
|
||
"duration": item.get("duration", 0),
|
||
"startSecond": item.get("startSecond"),
|
||
"endSecond": item.get("endSecond"),
|
||
"collectionId": item.get("collectionId", ""),
|
||
"meta": item.get("meta", {}),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
"fileCount": len(package_files),
|
||
"pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1"],
|
||
"storagePath": str(package_dir),
|
||
"quality": quality_payload,
|
||
"annotation": annotation_payload,
|
||
}
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-packages",
|
||
"packageCount": len(packages),
|
||
"items": packages,
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "packages-manifest.json", manifest, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "packages", node_data=node_data, output=output_binding(node_data)))
|
||
return artifacts
|
||
|
||
|
||
def builtin_audit_sampling(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "packageCount", "items")
|
||
if payload is None:
|
||
raise RuntimeError("抽检采样需要上游成品数据输入")
|
||
rules = normalize_sampling_rules(parse_json_param(params, "RULES_JSON", [], list))
|
||
sampled_items = []
|
||
summaries = []
|
||
for rule in rules:
|
||
rule_id = rule["id"]
|
||
field_path = rule["field"]
|
||
operator = rule["operator"]
|
||
value = rule.get("value")
|
||
percent = rule["samplePercent"]
|
||
matched = []
|
||
sampled = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
if not compare_values(get_field_value(item, field_path), operator, value):
|
||
continue
|
||
matched.append(item)
|
||
package_name = str(item.get("packageName", "")).strip() or str(item.get("clipId", "")).strip()
|
||
if stable_sample_hit(f"{rule_id}:{package_name}", percent):
|
||
sampled.append(item)
|
||
sampled_items.append({**item, "sampleRuleId": rule_id})
|
||
summaries.append(
|
||
{
|
||
"id": rule_id,
|
||
"field": field_path,
|
||
"operator": operator,
|
||
"value": value,
|
||
"samplePercent": percent,
|
||
"matchedCount": len(matched),
|
||
"sampledCount": len(sampled),
|
||
}
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-audit-samples",
|
||
"rules": summaries,
|
||
"sampledCount": len(sampled_items),
|
||
"items": sampled_items,
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "audit-samples.json", manifest, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_condition_router(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_path = params.get("FIELD_PATH", "").strip()
|
||
operator = normalize_comparison_operator(params.get("OPERATOR", "eq"))
|
||
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
|
||
rows = collect_items(payloads_from_inputs(inputs))
|
||
matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
|
||
unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)]
|
||
outputs = node_data.get("outputs", []) or [
|
||
{"id": "matched", "name": "matched", "type": "json"},
|
||
{"id": "unmatched", "name": "unmatched", "type": "json"},
|
||
]
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json",
|
||
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)},
|
||
str(outputs[0].get("type", "json")),
|
||
output=outputs[0],
|
||
),
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json",
|
||
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)},
|
||
str(outputs[1].get("type", "json")),
|
||
output=outputs[1],
|
||
),
|
||
]
|
||
|
||
|
||
def builtin_lerobot_summary(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent)
|
||
payload = summarize_lerobot_root(root)
|
||
payload["sourcePath"] = str(source_path)
|
||
elif inputs:
|
||
payload = summarize_lerobot_from_inputs(inputs)
|
||
payload["sourcePath"] = ""
|
||
else:
|
||
raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录")
|
||
payload["datasetType"] = "lerobot"
|
||
return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_rosbag_summary(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
if source_path.is_dir():
|
||
payload = {
|
||
"datasetType": "rosbag2",
|
||
"sourcePath": str(source_path),
|
||
"summary": summarize_directory_tree(source_path),
|
||
}
|
||
metadata_yaml = source_path / "metadata.yaml"
|
||
if metadata_yaml.exists():
|
||
payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"]
|
||
db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()]
|
||
if db3_files:
|
||
payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"]
|
||
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
||
payload = summarize_rosbag_file(source_path)
|
||
payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag")
|
||
payload["sourcePath"] = str(source_path)
|
||
else:
|
||
for artifact in inputs:
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}:
|
||
payload = {
|
||
"datasetType": preview.get("format", "rosbag"),
|
||
"sourcePath": str(artifact.get("path", "")),
|
||
"preview": preview,
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
||
raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件")
|
||
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_generic(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_label: str,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = [load_artifact_payload(item) for item in inputs]
|
||
summary = {
|
||
"nodeLabel": node_label,
|
||
"inputCount": len(inputs),
|
||
"inputs": [summarize_json_like(item) for item in payloads],
|
||
"config": {
|
||
"bucket": node_data.get("bucket", ""),
|
||
"prefix": node_data.get("prefix", ""),
|
||
"pattern": node_data.get("pattern", ""),
|
||
},
|
||
}
|
||
outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}]
|
||
artifacts: list[dict[str, Any]] = []
|
||
for index, output in enumerate(outputs):
|
||
output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}"
|
||
filename = f"{output_name}.json"
|
||
payload = {**summary, "outputName": output_name}
|
||
artifacts.append(
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
filename,
|
||
payload,
|
||
str(output.get("type", "json")),
|
||
output=output,
|
||
)
|
||
)
|
||
return artifacts
|
||
|
||
|
||
def builtin_model_inference(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('model_inference', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_vla_model_train(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('vla_model_train', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_sim3d_asset_generate(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('sim3d_asset_generate', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_holobrain_data_prepare(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('holobrain_data_prepare', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_holobrain_train(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('holobrain_train', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_holobrain_export(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('holobrain_export', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_holobrain_robotwin_eval(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('holobrain_robotwin_eval', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_holobrain_inference_test(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('holobrain_inference_test', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_vla_model_inference(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('vla_model_inference', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_vla_model_evaluate(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('vla_model_evaluate', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def builtin_shell_command(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
return run_operator_module('shell_command', run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
|
||
def execute_command(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
command: str,
|
||
) -> list[dict[str, Any]]:
|
||
validate_command_policy(command)
|
||
completed = subprocess.run(
|
||
command,
|
||
cwd=output_dir,
|
||
shell=True,
|
||
text=True,
|
||
capture_output=True,
|
||
env=os.environ.copy(),
|
||
check=False,
|
||
)
|
||
for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)):
|
||
for line in content.splitlines():
|
||
if line.strip():
|
||
log(f"[operator-runner][command][{stream_name}] {line}")
|
||
if completed.returncode != 0:
|
||
raise RuntimeError(f"command failed with exit code {completed.returncode}")
|
||
artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data)
|
||
if artifacts:
|
||
return artifacts
|
||
payload = {"stdout": completed.stdout, "stderr": completed.stderr}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
OPERATOR_HELPERS = {
|
||
"TABULAR_FILE_SUFFIXES": TABULAR_FILE_SUFFIXES,
|
||
"DEFAULT_MAX_HAND_OCCLUSION": DEFAULT_MAX_HAND_OCCLUSION,
|
||
"DEFAULT_MAX_SHAKE": DEFAULT_MAX_SHAKE,
|
||
"DEFAULT_MIN_CLARITY": DEFAULT_MIN_CLARITY,
|
||
"RESERVED_MONGO_COLLECTIONS": RESERVED_MONGO_COLLECTIONS,
|
||
"VALID_SOURCE_DATA_SOURCES": VALID_SOURCE_DATA_SOURCES,
|
||
"VIDEO_FILE_SUFFIXES": VIDEO_FILE_SUFFIXES,
|
||
"apply_incremental_source_filter": apply_incremental_source_filter,
|
||
"artifact_sha256": artifact_sha256,
|
||
"analyze_video_quality_metrics": analyze_video_quality_metrics,
|
||
"annotate_video_clip": annotate_video_clip,
|
||
"artifact_preview_payload": artifact_preview_payload,
|
||
"bos_key_join": bos_key_join,
|
||
"bos_prefix": bos_prefix,
|
||
"build_bos_client": build_bos_client,
|
||
"build_mongo_client": build_mongo_client,
|
||
"build_artifact": build_artifact,
|
||
"build_virtual_artifact": build_virtual_artifact,
|
||
"canonical_bos_path": canonical_bos_path,
|
||
"clean_metadata": clean_metadata,
|
||
"collect_items": collect_items,
|
||
"collect_output_artifacts": collect_output_artifacts,
|
||
"common_input_root": common_input_root,
|
||
"compare_values": compare_values,
|
||
"date_in_range": date_in_range,
|
||
"documents_from_inputs": documents_from_inputs,
|
||
"execute_command": execute_command,
|
||
"extract_preview_data": extract_preview_data,
|
||
"find_lerobot_root": find_lerobot_root,
|
||
"ffmpeg_available": ffmpeg_available,
|
||
"flatten_context": flatten_context,
|
||
"first_payload_with_keys": first_payload_with_keys,
|
||
"float_value": float_value,
|
||
"get_field_value": get_field_value,
|
||
"hyperparams": hyperparams,
|
||
"incremental_directory_depth": incremental_directory_depth,
|
||
"incremental_unit": incremental_unit,
|
||
"infer_port_type": infer_port_type,
|
||
"inspect_video_file": inspect_video_file,
|
||
"json_compatible": json_compatible,
|
||
"list_bos_entries": list_bos_entries,
|
||
"list_source_entries": list_source_entries,
|
||
"load_artifact_payload": load_artifact_payload,
|
||
"load_tabular_rows": load_tabular_rows,
|
||
"load_payload_from_file": load_payload_from_file,
|
||
"load_xplatform_index_payload": load_xplatform_index_payload,
|
||
"load_json_file": load_json_file,
|
||
"log": log,
|
||
"map_json_row": map_json_row,
|
||
"materialize_clip_from_source": materialize_clip_from_source,
|
||
"materialize_package_artifacts_from_manifest": materialize_package_artifacts_from_manifest,
|
||
"materialized_upload_source": materialized_upload_source,
|
||
"merge_dicts": merge_dicts,
|
||
"normalize_json_payload": normalize_json_payload,
|
||
"normalize_date_value": normalize_date_value,
|
||
"normalize_comparison_operator": normalize_comparison_operator,
|
||
"normalize_field_map": normalize_field_map,
|
||
"normalize_sampling_rules": normalize_sampling_rules,
|
||
"output_binding": output_binding,
|
||
"package_config_payload": package_config_payload,
|
||
"package_related_artifact_names": package_related_artifact_names,
|
||
"parse_bool_param": parse_bool_param,
|
||
"parse_bos_path": parse_bos_path,
|
||
"parse_json_param": parse_json_param,
|
||
"parse_positive_float_param": parse_positive_float_param,
|
||
"parse_positive_int_param": parse_positive_int_param,
|
||
"parse_unit_interval_param": parse_unit_interval_param,
|
||
"payloads_from_inputs": payloads_from_inputs,
|
||
"preview_payload_for_remote_entry": preview_payload_for_remote_entry,
|
||
"preview_text_payload": preview_text_payload,
|
||
"render_template_text": render_template_text,
|
||
"resolve_source_path": resolve_source_path,
|
||
"rows_from_payloads": rows_from_payloads,
|
||
"source_path_from_item": source_path_from_item,
|
||
"source_entry_artifact": source_entry_artifact,
|
||
"source_ref_from_artifact": source_ref_from_artifact,
|
||
"stable_sample_hit": stable_sample_hit,
|
||
"summarize_directory_tree": summarize_directory_tree,
|
||
"summarize_input_artifacts": summarize_input_artifacts,
|
||
"summarize_lerobot_from_inputs": summarize_lerobot_from_inputs,
|
||
"summarize_lerobot_root": summarize_lerobot_root,
|
||
"summarize_rosbag_file": summarize_rosbag_file,
|
||
"summarize_sqlite_payload": summarize_sqlite_payload,
|
||
"text_value": text_value,
|
||
"upload_relative_path": upload_relative_path,
|
||
"write_csv_artifact": write_csv_artifact,
|
||
"write_json_artifact": write_json_artifact,
|
||
"write_text_artifact": write_text_artifact,
|
||
"write_video_segment": write_video_segment,
|
||
"write_video_segment_with_ffmpeg": write_video_segment_with_ffmpeg,
|
||
"write_video_segments": write_video_segments,
|
||
}
|
||
|
||
|
||
BuiltinHandler = Any
|
||
_OPERATOR_MODULE_CACHE: dict[str, Any] = {}
|
||
def load_operator_module(slug: str) -> Any:
|
||
if not re.fullmatch(r"[a-z0-9_]+", slug):
|
||
raise ValueError(f"invalid operator slug: {slug}")
|
||
source = PROJECT_ROOT / "operator-repos" / "builtin" / slug / "src" / "operator.py"
|
||
module = _OPERATOR_MODULE_CACHE.get(slug)
|
||
if module is None:
|
||
spec = importlib.util.spec_from_file_location("droboticflow_operator_" + slug, source)
|
||
if spec is None or spec.loader is None:
|
||
raise RuntimeError(f"cannot load operator source: {source}")
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
_OPERATOR_MODULE_CACHE[slug] = module
|
||
return module
|
||
|
||
|
||
def run_operator_module(
|
||
module_path: str,
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
source_path = os.getenv("DROBOTICFLOW_OPERATOR_SOURCE", "").strip()
|
||
if source_path:
|
||
source = pathlib.Path(source_path)
|
||
if not source.is_file():
|
||
raise RuntimeError(f"DROBOTICFLOW_OPERATOR_SOURCE does not exist: {source}")
|
||
module_name = "droboticflow_split_operator_" + hashlib.sha1(str(source).encode("utf-8")).hexdigest()
|
||
spec = importlib.util.spec_from_file_location(module_name, source)
|
||
if spec is None or spec.loader is None:
|
||
raise RuntimeError(f"cannot load operator source: {source}")
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module.run(run_id, node_id, output_dir, node_data, inputs, OPERATOR_HELPERS)
|
||
return load_operator_module(module_path).run(run_id, node_id, output_dir, node_data, inputs, OPERATOR_HELPERS)
|
||
|
||
|
||
def operator_handler(module_path: str) -> BuiltinHandler:
|
||
def run(run_id: str, node_id: str, output_dir: pathlib.Path, node_data: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
return run_operator_module(module_path, run_id, node_id, output_dir, node_data, inputs)
|
||
|
||
return run
|
||
|
||
|
||
def builtin_generic_handler(run_id: str, node_id: str, output_dir: pathlib.Path, node_label: str, node_data: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
return builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs)
|
||
|
||
|
||
BUILTIN_HANDLERS: dict[str, BuiltinHandler] = {
|
||
"builtin_source": operator_handler('real_source'),
|
||
"source": operator_handler('real_source'),
|
||
"builtin_act_policy_train": operator_handler('act_policy_train'),
|
||
"builtin_bos_incremental_scan": operator_handler('bos_incremental_scan'),
|
||
"builtin_merge": operator_handler('merge'),
|
||
"builtin_intersection": builtin_intersection,
|
||
"builtin_distribution": builtin_distribution,
|
||
"builtin_json_extract": operator_handler('json_field_extract'),
|
||
"builtin_json_filter": operator_handler('json_filter'),
|
||
"builtin_json_map": operator_handler('json_map_rename'),
|
||
"builtin_mongodb_write": operator_handler('mongodb_write'),
|
||
"builtin_text_template": operator_handler('template_render'),
|
||
"builtin_csv_to_json": operator_handler('csv_to_json'),
|
||
"builtin_json_to_csv": operator_handler('json_to_csv'),
|
||
"builtin_directory_scan": operator_handler('directory_scan'),
|
||
"builtin_diffusion_policy_train": operator_handler('diffusion_policy_train'),
|
||
"builtin_xplatform_index": operator_handler('xplatform_dataset_index'),
|
||
"builtin_collection_validate": operator_handler('xplatform_dataset_validate'),
|
||
"builtin_video_quality": operator_handler('video_quality'),
|
||
"builtin_dataset_filter": operator_handler('video_filter'),
|
||
"builtin_video_slicing": operator_handler('video_slice'),
|
||
"builtin_video_target_clip": operator_handler('video_target_clip'),
|
||
"builtin_video_annotation": operator_handler('video_annotate'),
|
||
"builtin_result_package": operator_handler('package'),
|
||
"builtin_audit_sampling": operator_handler('audit_sample'),
|
||
"builtin_hf_dataset_probe": operator_handler('hf_dataset_probe'),
|
||
"builtin_agibot_world2026_validate": operator_handler('agibot_world2026_validate'),
|
||
"builtin_rlds_episode_summary": operator_handler('rlds_episode_summary'),
|
||
"builtin_bridgedata_tfds_manifest_validate": operator_handler('bridgedata_tfds_manifest_validate'),
|
||
"builtin_droid_raw_episode_validate": operator_handler('droid_raw_episode_validate'),
|
||
"builtin_libero_hdf5_task_validate": operator_handler('libero_hdf5_task_validate'),
|
||
"builtin_hdf5_demonstration_summary": operator_handler('hdf5_demonstration_summary'),
|
||
"builtin_aloha_hdf5_validate": operator_handler('aloha_hdf5_validate'),
|
||
"builtin_robomimic_hdf5_validate": operator_handler('robomimic_hdf5_validate'),
|
||
"builtin_robocasa_hdf5_demonstration_validate": operator_handler('robocasa_hdf5_demonstration_validate'),
|
||
"builtin_furniturebench_pkl_demonstration_validate": operator_handler('furniturebench_pkl_demonstration_validate'),
|
||
"builtin_roboturk_real_hdf5_validate": operator_handler('roboturk_real_hdf5_validate'),
|
||
"builtin_roboset_hdf5_validate": operator_handler('roboset_hdf5_validate'),
|
||
"builtin_rh20t_scene_validate": operator_handler('rh20t_scene_validate'),
|
||
"builtin_robomind_hdf5_quality_validate": operator_handler('robomind_hdf5_quality_validate'),
|
||
"vla_sample_manifest_validate": operator_handler('vla_sample_manifest_validate'),
|
||
"builtin_rlbench_dataset_validate": operator_handler('rlbench_dataset_validate'),
|
||
"builtin_maniskill_trajectory_validate": operator_handler('maniskill_trajectory_validate'),
|
||
"builtin_calvin_episode_validate": operator_handler('calvin_episode_validate'),
|
||
"builtin_rtx_action_schema_validate": operator_handler('rtx_action_schema_validate'),
|
||
"builtin_embodiment_metadata_validate": operator_handler('embodiment_metadata_validate'),
|
||
"builtin_force_torque_sensor_validate": operator_handler('force_torque_sensor_validate'),
|
||
"builtin_lerobot_video_alignment_validate": operator_handler('lerobot_video_alignment_validate'),
|
||
"builtin_lerobot_v3_episode_index_validate": operator_handler('lerobot_v3_episode_index_validate'),
|
||
"builtin_lerobot_dataset_validate": operator_handler('lerobot_dataset_validate'),
|
||
"builtin_lerobot_metadata_consistency_validate": operator_handler('lerobot_metadata_consistency_validate'),
|
||
"builtin_trajectory_integrity_validate": operator_handler('trajectory_integrity_validate'),
|
||
"builtin_action_state_quality_validate": operator_handler('action_state_quality_validate'),
|
||
"builtin_episode_boundary_validate": operator_handler('episode_boundary_validate'),
|
||
"builtin_joint_limit_validate": operator_handler('joint_limit_validate'),
|
||
"builtin_dataset_split_leakage_validate": operator_handler('dataset_split_leakage_validate'),
|
||
"builtin_dataset_split_manifest": operator_handler('dataset_split_manifest'),
|
||
"builtin_media_reference_validate": operator_handler('media_reference_validate'),
|
||
"builtin_dataset_schema_contract_validate": operator_handler('dataset_schema_contract_validate'),
|
||
"builtin_mcap_topic_summary": operator_handler('mcap_topic_summary'),
|
||
"builtin_tfrecord_structural_validate": operator_handler('tfrecord_structural_validate'),
|
||
"builtin_dataset_provenance_audit": operator_handler('dataset_provenance_audit'),
|
||
"builtin_graspnet_dataset_validate": operator_handler('graspnet_dataset_validate'),
|
||
"builtin_bop_dataset_validate": operator_handler('bop_dataset_validate'),
|
||
"builtin_dexycb_dataset_validate": operator_handler('dexycb_dataset_validate'),
|
||
"builtin_ho3d_dataset_validate": operator_handler('ho3d_dataset_validate'),
|
||
"builtin_wilor_hand_pose": operator_handler('wilor_hand_pose'),
|
||
"builtin_hand_removal": operator_handler('hand_removal'),
|
||
"builtin_webdataset_shard_summary": operator_handler('webdataset_shard_summary'),
|
||
"builtin_umi_zarr_dataset_validate": operator_handler('umi_zarr_dataset_validate'),
|
||
"builtin_zarr_trajectory_summary": operator_handler('zarr_trajectory_summary'),
|
||
"builtin_camera_calibration_validate": operator_handler('camera_calibration_validate'),
|
||
"builtin_tf_transform_tree_validate": operator_handler('tf_transform_tree_validate'),
|
||
"builtin_ros_joint_state_validate": operator_handler('ros_joint_state_validate'),
|
||
"builtin_hand_object_pose_validate": operator_handler('hand_object_pose_validate'),
|
||
"builtin_partnet_mobility_asset_validate": operator_handler('partnet_mobility_asset_validate'),
|
||
"builtin_mjcf_actuator_validate": operator_handler('mjcf_actuator_validate'),
|
||
"builtin_robot_model_asset_validate": operator_handler('robot_model_asset_validate'),
|
||
"builtin_pointcloud_summary": operator_handler('pointcloud_summary'),
|
||
"builtin_sensor_timestamp_sync_validate": operator_handler('sensor_timestamp_sync_validate'),
|
||
"builtin_parquet_table_summary": operator_handler('parquet_table_summary'),
|
||
"builtin_coco_annotation_validate": operator_handler('coco_annotation_validate'),
|
||
"builtin_yolo_detection_validate": operator_handler('yolo_detection_validate'),
|
||
"builtin_rgbd_frame_pair_validate": operator_handler('rgbd_frame_pair_validate'),
|
||
"builtin_segmentation_mask_validate": operator_handler('segmentation_mask_validate'),
|
||
"builtin_language_instruction_validate": operator_handler('language_instruction_validate'),
|
||
"builtin_condition_router": operator_handler('conditional_route'),
|
||
"builtin_human_review": operator_handler('human_review'),
|
||
"lerobot": operator_handler('lerobot_summary'),
|
||
"rosbag": operator_handler('rosbag_summary'),
|
||
"builtin_upload": operator_handler('upload'),
|
||
"builtin_holobrain_data_prepare": operator_handler('holobrain_data_prepare'),
|
||
"builtin_holobrain_train": operator_handler('holobrain_train'),
|
||
"builtin_holobrain_export": operator_handler('holobrain_export'),
|
||
"builtin_holobrain_robotwin_eval": operator_handler('holobrain_robotwin_eval'),
|
||
"builtin_holobrain_inference_test": operator_handler('holobrain_inference_test'),
|
||
"builtin_openvla_finetune": operator_handler('openvla_finetune'),
|
||
"builtin_openvla_inference": operator_handler('openvla_inference'),
|
||
"vla_model_train": operator_handler('vla_model_train'),
|
||
"sim3d_asset_generate": operator_handler('sim3d_asset_generate'),
|
||
"vla_model_inference": operator_handler('vla_model_inference'),
|
||
"vla_model_evaluate": operator_handler('vla_model_evaluate'),
|
||
"builtin_model_inference": builtin_model_inference,
|
||
"builtin_yolo_embodied_detection_train": operator_handler('yolo_embodied_detection_train'),
|
||
"builtin_yolo_video_target_clip": operator_handler('yolo_video_target_clip'),
|
||
"builtin_shell_command": builtin_shell_command,
|
||
"builtin_generic": builtin_generic_handler,
|
||
}
|
||
def runtime_handler() -> str:
|
||
handler = os.getenv("DROBOTICFLOW_OPERATOR_HANDLER", "").strip()
|
||
if not handler:
|
||
raise RuntimeError("DROBOTICFLOW_OPERATOR_HANDLER is required; label-based dispatch is not supported")
|
||
if handler not in BUILTIN_HANDLERS:
|
||
known = ", ".join(sorted(BUILTIN_HANDLERS))
|
||
raise RuntimeError(f"DROBOTICFLOW_OPERATOR_HANDLER={handler!r} is not registered; known handlers: {known}")
|
||
return handler
|
||
|
||
|
||
def main() -> int:
|
||
run_id = os.getenv("RUN_ID", "").strip()
|
||
workflow_id = os.getenv("WORKFLOW_ID", "").strip()
|
||
node_id = os.getenv("NODE_ID", "").strip()
|
||
node_label = os.getenv("NODE_LABEL", "").strip() or node_id
|
||
output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id)))
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
node_data = load_json_env("NODE_DATA_JSON", {})
|
||
inputs = load_input_artifacts()
|
||
input_bindings = load_input_bindings()
|
||
started_at = now_iso()
|
||
start_cpu, start_memory = resource_snapshot()
|
||
sampler = ResourceSampler(start_cpu, start_memory, started_at)
|
||
sampler.start()
|
||
|
||
logs: list[dict[str, Any]] = []
|
||
|
||
def append_log(level: str, message: str) -> None:
|
||
logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label})
|
||
log(f"[operator-runner][{level.lower()}] {message}")
|
||
|
||
append_log(
|
||
"INFO",
|
||
f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir} input_bindings={len(input_bindings)}",
|
||
)
|
||
append_log("INFO", "Preparing operator execution context")
|
||
|
||
try:
|
||
params = hyperparams(node_data)
|
||
command = params.get("COMMAND") or params.get("SCRIPT")
|
||
artifacts: list[dict[str, Any]]
|
||
if command:
|
||
append_log("INFO", "Executing explicit command")
|
||
artifacts = execute_command(run_id, node_id, output_dir, node_data, command)
|
||
else:
|
||
handler = runtime_handler()
|
||
append_log("INFO", f"Selected built-in handler: {handler}")
|
||
builtin_handler = BUILTIN_HANDLERS[handler]
|
||
if handler == "builtin_generic":
|
||
artifacts = builtin_handler(run_id, node_id, output_dir, node_label, node_data, inputs)
|
||
else:
|
||
artifacts = builtin_handler(run_id, node_id, output_dir, node_data, inputs)
|
||
finally:
|
||
ended_at = now_iso()
|
||
resource_samples = sampler.stop(ended_at)
|
||
|
||
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
|
||
data_metrics = infer_data_metrics(node_data, artifacts, inputs)
|
||
result = {
|
||
"logs": logs,
|
||
"artifacts": artifacts,
|
||
"resourceSeries": resource_samples,
|
||
}
|
||
if data_metrics:
|
||
result["dataMetrics"] = data_metrics
|
||
stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip()
|
||
if stage_manifest_path:
|
||
write_json_file(
|
||
stage_manifest_path,
|
||
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
|
||
)
|
||
else:
|
||
write_json_file(
|
||
default_stage_manifest_path(output_dir),
|
||
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
|
||
)
|
||
|
||
result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip()
|
||
if result_output_path:
|
||
written_path = write_json_file(result_output_path, result)
|
||
print(RESULT_PATH_PREFIX + str(written_path), flush=True)
|
||
return 0
|
||
|
||
default_result_path = write_json_file(default_result_output_path(output_dir), result)
|
||
print(RESULT_PATH_PREFIX + str(default_result_path), flush=True)
|
||
return 0
|
||
|
||
|
||
def tempfile_dir(run_id: str, node_id: str) -> str:
|
||
base = pathlib.Path("/tmp/droboticflow")
|
||
return str(base / (run_id or "run") / (node_id or "node"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
sys.exit(main())
|
||
except Exception as error: # pragma: no cover
|
||
log(f"[operator-runner][error] {error}")
|
||
sys.exit(1)
|