forked from longtao.wu/droboticflow
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
4683 lines
190 KiB
Python
4683 lines
190 KiB
Python
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import datetime as dt
|
||
import contextlib
|
||
import fnmatch
|
||
import hashlib
|
||
import json
|
||
import mimetypes
|
||
import os
|
||
import pathlib
|
||
import base64
|
||
import csv
|
||
import math
|
||
import re
|
||
import resource
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import tarfile
|
||
import time
|
||
import zipfile
|
||
from typing import Any
|
||
from urllib import error, request
|
||
from urllib.parse import urlparse
|
||
|
||
|
||
RESULT_PREFIX = "DROBOTICFLOW_RESULT="
|
||
RESULT_PATH_PREFIX = "DROBOTICFLOW_RESULT_PATH="
|
||
TEXT_FILE_SUFFIXES = {
|
||
".txt",
|
||
".log",
|
||
".md",
|
||
".yaml",
|
||
".yml",
|
||
".xml",
|
||
".html",
|
||
".htm",
|
||
}
|
||
TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"}
|
||
IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
|
||
VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
|
||
AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"}
|
||
ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"}
|
||
ARCHIVE_FILE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip")
|
||
XPLATFORM_VIDEO_META_FILES = ("video_meta.json", "video_metadata.json")
|
||
XPLATFORM_INLINE_JSON_FILES = {"meta.json", "intrinsics.json", *XPLATFORM_VIDEO_META_FILES}
|
||
PREVIEW_TEXT_LIMIT = 12_000
|
||
PREVIEW_TEXT_LINE_LIMIT = 240
|
||
PREVIEW_TABLE_ROW_LIMIT = 24
|
||
PREVIEW_FILE_LIST_LIMIT = 40
|
||
PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024
|
||
DATASET_PARTITION_SIZE = 500
|
||
QUALITY_SAMPLE_LIMIT = 48
|
||
QUALITY_ANALYSIS_MAX_DIMENSION = 640
|
||
QUALITY_SKIN_COMPONENT_MIN_RATIO = 0.002
|
||
RESERVED_MONGO_COLLECTIONS = {
|
||
"projects",
|
||
"operators",
|
||
"workflows",
|
||
"templates",
|
||
"runs",
|
||
"sourceStates",
|
||
"auditTasks",
|
||
"samplingTasks",
|
||
"users",
|
||
"storageConnections",
|
||
"settings",
|
||
}
|
||
DATASET_PARTITION_KEYS = (
|
||
"items",
|
||
"entries",
|
||
"rows",
|
||
"collections",
|
||
"catalogCollections",
|
||
"catalogItems",
|
||
"catalogEntries",
|
||
)
|
||
PAYLOAD_FILE_READ_ATTEMPTS = 60
|
||
PAYLOAD_FILE_READ_DELAY_SECONDS = 1.0
|
||
|
||
|
||
def now_iso() -> str:
|
||
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||
|
||
|
||
def log(message: str) -> None:
|
||
print(message, flush=True)
|
||
|
||
|
||
def load_json_env(name: str, default: Any) -> Any:
|
||
value = os.getenv(name, "").strip()
|
||
if not value:
|
||
return default
|
||
return json.loads(value)
|
||
|
||
|
||
def load_json_file(path: str, default: Any) -> Any:
|
||
raw = str(path or "").strip()
|
||
if not raw:
|
||
return default
|
||
candidate = pathlib.Path(raw)
|
||
if not candidate.exists():
|
||
return default
|
||
return json.loads(candidate.read_text(encoding="utf-8"))
|
||
|
||
|
||
def write_json_file(path: str | pathlib.Path, value: Any) -> pathlib.Path:
|
||
candidate = pathlib.Path(path)
|
||
candidate.parent.mkdir(parents=True, exist_ok=True)
|
||
candidate.write_text(json.dumps(json_compatible(value), ensure_ascii=False), encoding="utf-8")
|
||
return candidate
|
||
|
||
|
||
def text_value(value: Any) -> str:
|
||
if isinstance(value, bytes):
|
||
return value.decode("utf-8", errors="replace")
|
||
return str(value or "")
|
||
|
||
|
||
def json_compatible(value: Any) -> Any:
|
||
if isinstance(value, bytes):
|
||
return value.decode("utf-8", errors="replace")
|
||
if isinstance(value, dict):
|
||
return {str(key): json_compatible(item) for key, item in value.items()}
|
||
if isinstance(value, list):
|
||
return [json_compatible(item) for item in value]
|
||
if isinstance(value, tuple):
|
||
return [json_compatible(item) for item in value]
|
||
return value
|
||
|
||
|
||
def load_input_artifacts() -> list[dict[str, Any]]:
|
||
input_artifacts_path = os.getenv("INPUT_ARTIFACTS_PATH", "").strip()
|
||
if input_artifacts_path:
|
||
loaded = load_json_file(input_artifacts_path, [])
|
||
if isinstance(loaded, list):
|
||
return json_compatible(loaded)
|
||
loaded = load_json_env("INPUT_ARTIFACTS_JSON", [])
|
||
if isinstance(loaded, list):
|
||
return json_compatible(loaded)
|
||
return []
|
||
|
||
|
||
def stage_manifest_payload(
|
||
run_id: str,
|
||
workflow_id: str,
|
||
node_id: str,
|
||
node_label: str,
|
||
output_dir: pathlib.Path,
|
||
artifacts: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
manifest_artifacts: list[dict[str, Any]] = []
|
||
data_refs: list[dict[str, Any]] = []
|
||
for artifact in artifacts:
|
||
item = {
|
||
"id": text_value(artifact.get("id")),
|
||
"name": text_value(artifact.get("name")),
|
||
"type": text_value(artifact.get("type")),
|
||
"path": text_value(artifact.get("path")),
|
||
"size": text_value(artifact.get("size")),
|
||
}
|
||
if text_value(artifact.get("portId")):
|
||
item["portId"] = text_value(artifact.get("portId"))
|
||
if text_value(artifact.get("portName")):
|
||
item["portName"] = text_value(artifact.get("portName"))
|
||
if isinstance(artifact.get("metadata"), dict) and artifact["metadata"]:
|
||
item["metadata"] = json_compatible(artifact["metadata"])
|
||
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
|
||
if data_ref:
|
||
item["dataRef"] = json_compatible(data_ref)
|
||
data_refs.append(
|
||
{
|
||
"artifactId": item["id"],
|
||
"artifactName": item["name"],
|
||
"portId": item.get("portId", ""),
|
||
**json_compatible(data_ref),
|
||
}
|
||
)
|
||
manifest_artifacts.append(item)
|
||
return {
|
||
"runId": run_id,
|
||
"workflowId": workflow_id,
|
||
"nodeId": node_id,
|
||
"nodeLabel": node_label,
|
||
"outputDir": str(output_dir),
|
||
"artifactCount": len(manifest_artifacts),
|
||
"artifacts": manifest_artifacts,
|
||
"dataRefs": data_refs,
|
||
"primaryDataRefs": data_refs,
|
||
"generatedAt": now_iso(),
|
||
}
|
||
|
||
|
||
def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path:
|
||
return output_dir.parent / "result.json"
|
||
|
||
|
||
def default_stage_manifest_path(output_dir: pathlib.Path) -> pathlib.Path:
|
||
return output_dir.parent / "stage-manifest.json"
|
||
|
||
|
||
def normalize_json_payload(value: Any) -> dict[str, Any]:
|
||
if isinstance(value, dict):
|
||
return json_compatible(value)
|
||
if isinstance(value, list):
|
||
return {"items": json_compatible(value), "itemCount": len(value)}
|
||
return {"value": json_compatible(value)}
|
||
|
||
|
||
def bos_prefix(value: str) -> str:
|
||
return str(value or "").strip().strip("/")
|
||
|
||
|
||
def canonical_bos_path(bucket: str, key: str) -> str:
|
||
normalized_bucket = str(bucket or "").strip()
|
||
normalized_key = str(key or "").strip().lstrip("/")
|
||
if not normalized_bucket:
|
||
return ""
|
||
if not normalized_key:
|
||
return f"bos://{normalized_bucket}"
|
||
return f"bos://{normalized_bucket}/{normalized_key}"
|
||
|
||
|
||
def parse_bos_path(value: str) -> tuple[str, str]:
|
||
raw = str(value or "").strip()
|
||
if not raw.startswith("bos://"):
|
||
return "", ""
|
||
bucket_and_key = raw[len("bos://") :]
|
||
if not bucket_and_key:
|
||
return "", ""
|
||
bucket, _, key = bucket_and_key.partition("/")
|
||
return bucket.strip(), bos_prefix(key)
|
||
|
||
|
||
def is_xplatform_video_meta_name(value: str) -> bool:
|
||
return str(value or "").strip().lower() in XPLATFORM_VIDEO_META_FILES
|
||
|
||
|
||
def load_collection_video_meta(collection_dir: pathlib.Path) -> tuple[dict[str, dict[str, Any]], str]:
|
||
for candidate in XPLATFORM_VIDEO_META_FILES:
|
||
payload = optional_json_file(collection_dir / candidate)
|
||
if payload is not None:
|
||
return parse_video_meta_rows(payload), candidate
|
||
return {}, ""
|
||
|
||
|
||
def sampled_frame_positions(frame_count: int, sample_limit: int = QUALITY_SAMPLE_LIMIT) -> list[int]:
|
||
if frame_count <= 0:
|
||
return []
|
||
if frame_count <= sample_limit:
|
||
return list(range(frame_count))
|
||
last_frame = max(frame_count - 1, 0)
|
||
positions = {
|
||
min(last_frame, max(0, int(round(index * last_frame / max(sample_limit - 1, 1)))))
|
||
for index in range(sample_limit)
|
||
}
|
||
return sorted(positions)
|
||
|
||
|
||
def resize_quality_frame(frame: Any, max_dimension: int = QUALITY_ANALYSIS_MAX_DIMENSION) -> Any:
|
||
try:
|
||
height, width = frame.shape[:2]
|
||
except Exception:
|
||
return frame
|
||
longest_edge = max(width, height)
|
||
if longest_edge <= 0 or longest_edge <= max_dimension:
|
||
return frame
|
||
scale = max_dimension / float(longest_edge)
|
||
target_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
|
||
cv2, _ = require_cv2()
|
||
return cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA)
|
||
|
||
|
||
def build_skin_mask(frame: Any) -> Any:
|
||
cv2, np = require_cv2()
|
||
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
|
||
ycrcb_mask = cv2.inRange(
|
||
ycrcb,
|
||
np.array((0, 133, 77), dtype=np.uint8),
|
||
np.array((255, 173, 127), dtype=np.uint8),
|
||
)
|
||
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
|
||
hsv_mask = cv2.inRange(
|
||
hsv,
|
||
np.array((0, 20, 40), dtype=np.uint8),
|
||
np.array((25, 220, 255), dtype=np.uint8),
|
||
)
|
||
return cv2.bitwise_and(ycrcb_mask, hsv_mask)
|
||
|
||
|
||
def largest_skin_component_ratio(mask: Any, min_component_ratio: float = QUALITY_SKIN_COMPONENT_MIN_RATIO) -> float:
|
||
cv2, np = require_cv2()
|
||
if mask is None:
|
||
return 0.0
|
||
total_area = int(mask.shape[0] * mask.shape[1]) if len(mask.shape) >= 2 else 0
|
||
if total_area <= 0:
|
||
return 0.0
|
||
kernel = np.ones((5, 5), dtype=np.uint8)
|
||
cleaned = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
|
||
cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_CLOSE, kernel)
|
||
component_threshold = max(1, int(total_area * max(min_component_ratio, 0.0)))
|
||
count, _, stats, _ = cv2.connectedComponentsWithStats(cleaned)
|
||
largest_area = 0
|
||
for index in range(1, count):
|
||
area = int(stats[index, cv2.CC_STAT_AREA])
|
||
if area < component_threshold:
|
||
continue
|
||
if area > largest_area:
|
||
largest_area = area
|
||
return float(largest_area / total_area)
|
||
|
||
|
||
def motion_jitter_ratio(shifts: list[tuple[float, float]], diagonal: float) -> float:
|
||
if not shifts:
|
||
return 0.0
|
||
_, np = require_cv2()
|
||
normalized_diagonal = max(diagonal, 1.0)
|
||
dx_values = [item[0] for item in shifts]
|
||
dy_values = [item[1] for item in shifts]
|
||
median_dx = float(np.median(dx_values))
|
||
median_dy = float(np.median(dy_values))
|
||
jitters = [
|
||
math.sqrt((dx - median_dx) ** 2 + (dy - median_dy) ** 2) / normalized_diagonal
|
||
for dx, dy in shifts
|
||
]
|
||
return sum(jitters) / max(len(jitters), 1)
|
||
|
||
|
||
def bos_key_join(prefix: str, relative_path: str) -> str:
|
||
normalized_prefix = bos_prefix(prefix)
|
||
normalized_relative = str(relative_path or "").strip().lstrip("/")
|
||
if not normalized_prefix:
|
||
return normalized_relative
|
||
if not normalized_relative:
|
||
return normalized_prefix
|
||
return f"{normalized_prefix}/{normalized_relative}"
|
||
|
||
|
||
def require_bos_sdk():
|
||
try:
|
||
import baidubce.protocol as protocol # type: ignore
|
||
from baidubce.auth.bce_credentials import BceCredentials # type: ignore
|
||
from baidubce.bce_client_configuration import BceClientConfiguration # type: ignore
|
||
from baidubce.services.bos.bos_client import BosClient # type: ignore
|
||
except Exception as error: # pragma: no cover - runtime environment specific
|
||
raise RuntimeError("BOS 扫描需要 bce-python-sdk 运行时") from error
|
||
return protocol, BceCredentials, BceClientConfiguration, BosClient
|
||
|
||
|
||
def build_bos_client(endpoint: str, access_key_id: str, secret_access_key: str):
|
||
protocol_module, BceCredentials, BceClientConfiguration, BosClient = require_bos_sdk()
|
||
parsed = urlparse(endpoint if "://" in endpoint else f"https://{endpoint}")
|
||
scheme = (parsed.scheme or "https").lower()
|
||
host = parsed.netloc or parsed.path
|
||
if not host:
|
||
raise RuntimeError("BOS endpoint 不能为空")
|
||
config = BceClientConfiguration(
|
||
credentials=BceCredentials(access_key_id, secret_access_key),
|
||
endpoint=host,
|
||
protocol=protocol_module.HTTPS if scheme == "https" else protocol_module.HTTP,
|
||
)
|
||
return BosClient(config), scheme
|
||
|
||
|
||
def resource_snapshot() -> tuple[float, int]:
|
||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||
cpu_seconds = usage.ru_utime + usage.ru_stime
|
||
if sys.platform == "darwin":
|
||
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
|
||
else:
|
||
memory_mib = max(1, int(usage.ru_maxrss / 1024))
|
||
return cpu_seconds, memory_mib
|
||
|
||
|
||
def resource_series(start_cpu: float, start_memory: int, started_at: str) -> list[dict[str, Any]]:
|
||
end_cpu, end_memory = resource_snapshot()
|
||
ended_at = now_iso()
|
||
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
|
||
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
|
||
elapsed = max((ended - started).total_seconds(), 0.001)
|
||
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
|
||
return [
|
||
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
|
||
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, "gpu": 0},
|
||
]
|
||
|
||
|
||
def format_size(num_bytes: int) -> str:
|
||
if num_bytes < 1024:
|
||
return f"{num_bytes} B"
|
||
if num_bytes < 1024 * 1024:
|
||
return f"{num_bytes / 1024:.1f} KiB"
|
||
if num_bytes < 1024 * 1024 * 1024:
|
||
return f"{num_bytes / (1024 * 1024):.1f} MiB"
|
||
return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB"
|
||
|
||
|
||
MAX_SHA256_FILE_BYTES = 64 * 1024 * 1024
|
||
|
||
|
||
def sha256_file(path: pathlib.Path) -> str:
|
||
digest = hashlib.sha256()
|
||
with path.open("rb") as handle:
|
||
for chunk in iter(lambda: handle.read(65536), b""):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def artifact_sha256(path: pathlib.Path, size_bytes: int) -> tuple[str, dict[str, Any]]:
|
||
if size_bytes > MAX_SHA256_FILE_BYTES:
|
||
return "", {"sha256Skipped": True, "sha256SkipReason": "size-limit", "sha256SkipBytes": size_bytes}
|
||
return sha256_file(path), {}
|
||
|
||
|
||
def require_cv2() -> tuple[Any, Any]:
|
||
try:
|
||
import cv2 # type: ignore
|
||
import numpy as np # type: ignore
|
||
except Exception as error: # pragma: no cover - environment specific
|
||
raise RuntimeError("视频算子需要 opencv-python-headless 和 numpy 运行时") from error
|
||
return cv2, np
|
||
|
||
|
||
def data_url_for_path(path: pathlib.Path, mime_type: str) -> str:
|
||
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
||
return f"data:{mime_type};base64,{encoded}"
|
||
|
||
|
||
def decode_data_url(url: str) -> tuple[str, bytes] | None:
|
||
if not url.startswith("data:") or "," not in url:
|
||
return None
|
||
header, encoded = url.split(",", 1)
|
||
if ";base64" not in header:
|
||
return None
|
||
mime_type = header[5:].split(";", 1)[0] or "application/octet-stream"
|
||
try:
|
||
return mime_type, base64.b64decode(encoded)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def download_remote_source_to_path(url: str, path: pathlib.Path) -> pathlib.Path | None:
|
||
parsed = urlparse(url)
|
||
if parsed.scheme.lower() not in {"http", "https"}:
|
||
return None
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
remote_request = request.Request(
|
||
url,
|
||
headers={
|
||
"User-Agent": "DroboticFlow operator-runner/1.0",
|
||
},
|
||
)
|
||
try:
|
||
with request.urlopen(remote_request, timeout=300) as response, path.open("wb") as handle:
|
||
shutil.copyfileobj(response, handle, length=1024 * 1024)
|
||
except error.HTTPError as exc:
|
||
raise RuntimeError(f"下载远程输入失败({exc.code}): {path.name}") from exc
|
||
except error.URLError as exc:
|
||
raise RuntimeError(f"下载远程输入失败: {path.name}") from exc
|
||
return path
|
||
|
||
|
||
def download_remote_source(url: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
|
||
path = output_dir / "_materialized_sources" / default_name
|
||
return download_remote_source_to_path(url, path)
|
||
|
||
|
||
def workspace_cached_bos_source_path(raw_path: str, output_dir: pathlib.Path) -> pathlib.Path | None:
|
||
bucket, key = parse_bos_path(raw_path)
|
||
if not bucket or not key:
|
||
return None
|
||
workspace_root = str(os.getenv("WORKSPACE_ROOT", "")).strip()
|
||
if workspace_root:
|
||
root = pathlib.Path(workspace_root)
|
||
if root.is_absolute():
|
||
return root / "_bos_cache" / bucket / key
|
||
return output_dir / "_materialized_sources" / bucket / key
|
||
|
||
|
||
def download_bos_source(raw_path: str, default_name: str, output_dir: pathlib.Path) -> pathlib.Path | None:
|
||
target = workspace_cached_bos_source_path(raw_path, output_dir)
|
||
if target is None:
|
||
return None
|
||
if target.exists() and target.stat().st_size > 0:
|
||
return target
|
||
bucket, key = parse_bos_path(raw_path)
|
||
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
if not bucket or not key or not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
|
||
return None
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
temp_target = target.with_name(target.name + ".part")
|
||
with contextlib.suppress(FileNotFoundError):
|
||
temp_target.unlink()
|
||
try:
|
||
client, _ = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
||
client.get_object_to_file(bucket, key, str(temp_target))
|
||
temp_target.replace(target)
|
||
except Exception as exc:
|
||
with contextlib.suppress(FileNotFoundError):
|
||
temp_target.unlink()
|
||
raise RuntimeError(f"下载 BOS 输入失败: {default_name}") from exc
|
||
return target
|
||
|
||
|
||
def signed_bos_source_url(raw_path: str) -> str:
|
||
bucket, key = parse_bos_path(raw_path)
|
||
if not bucket or not key:
|
||
return ""
|
||
bos_endpoint = os.getenv("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
if not bos_endpoint or not bos_access_key_id or not bos_secret_access_key:
|
||
return ""
|
||
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
||
return text_value(
|
||
client.generate_pre_signed_url(
|
||
bucket,
|
||
key,
|
||
expiration_in_seconds=3600,
|
||
protocol=scheme,
|
||
)
|
||
).strip()
|
||
|
||
|
||
def remote_video_url_for_source(source: dict[str, Any]) -> str:
|
||
preview_url = str(source.get("previewURL", "")).strip()
|
||
parsed_preview = urlparse(preview_url)
|
||
if parsed_preview.scheme.lower() in {"http", "https"}:
|
||
return preview_url
|
||
raw_path = str(source.get("path", "")).strip()
|
||
if raw_path.startswith("bos://"):
|
||
return signed_bos_source_url(raw_path)
|
||
return ""
|
||
|
||
|
||
def is_archive_path(path: pathlib.Path) -> bool:
|
||
name = path.name.lower()
|
||
return any(name.endswith(suffix) for suffix in ARCHIVE_FILE_SUFFIXES)
|
||
|
||
|
||
def safe_extract_archive(path: pathlib.Path, destination: pathlib.Path) -> list[str]:
|
||
destination.mkdir(parents=True, exist_ok=True)
|
||
destination_resolved = destination.resolve()
|
||
extracted_files: list[str] = []
|
||
|
||
def ensure_child(member_name: str) -> None:
|
||
target = (destination / member_name).resolve()
|
||
if target != destination_resolved and destination_resolved not in target.parents:
|
||
raise RuntimeError(f"archive entry escapes target directory: {member_name}")
|
||
|
||
if path.name.lower().endswith(".zip"):
|
||
with zipfile.ZipFile(path) as archive:
|
||
for member in archive.infolist():
|
||
if not member.filename or member.is_dir():
|
||
continue
|
||
ensure_child(member.filename)
|
||
extracted_files.append(member.filename)
|
||
archive.extractall(destination)
|
||
return extracted_files
|
||
|
||
with tarfile.open(path, "r:*") as archive:
|
||
members = archive.getmembers()
|
||
for member in members:
|
||
if not member.name or member.isdir():
|
||
continue
|
||
ensure_child(member.name)
|
||
extracted_files.append(member.name)
|
||
archive.extractall(destination)
|
||
return extracted_files
|
||
|
||
|
||
def artifact_slug(value: str) -> str:
|
||
normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-")
|
||
return normalized or "artifact"
|
||
|
||
|
||
def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str:
|
||
suffix = path.suffix.lower()
|
||
if suffix == ".json":
|
||
return "json"
|
||
if suffix in VIDEO_FILE_SUFFIXES:
|
||
return "video"
|
||
if suffix in IMAGE_FILE_SUFFIXES:
|
||
return "image"
|
||
return fallback
|
||
|
||
|
||
def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None:
|
||
cleaned = {key: json_compatible(value) for key, value in metadata.items() if value not in ("", None, [], {})}
|
||
return cleaned or None
|
||
|
||
|
||
def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None:
|
||
if path.suffix.lower() != ".json":
|
||
return None
|
||
for attempt in range(PAYLOAD_FILE_READ_ATTEMPTS):
|
||
if not path.exists():
|
||
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
|
||
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
|
||
continue
|
||
return None
|
||
try:
|
||
with path.open("r", encoding="utf-8") as handle:
|
||
value = json.load(handle)
|
||
except (FileNotFoundError, OSError, json.JSONDecodeError):
|
||
if attempt + 1 < PAYLOAD_FILE_READ_ATTEMPTS:
|
||
time.sleep(PAYLOAD_FILE_READ_DELAY_SECONDS)
|
||
continue
|
||
return None
|
||
except Exception:
|
||
return None
|
||
if isinstance(value, dict):
|
||
return value
|
||
if isinstance(value, list):
|
||
return {"items": value, "itemCount": len(value)}
|
||
return {"value": value}
|
||
return None
|
||
|
||
|
||
def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None:
|
||
outputs = node_data.get("outputs", []) or []
|
||
if name:
|
||
for output in outputs:
|
||
if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name:
|
||
return output
|
||
if 0 <= index < len(outputs):
|
||
return outputs[index]
|
||
return None
|
||
|
||
|
||
def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]:
|
||
with path.open("rb") as handle:
|
||
raw = handle.read(PREVIEW_TEXT_LIMIT + 1)
|
||
truncated = len(raw) > PREVIEW_TEXT_LIMIT
|
||
text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace")
|
||
lines = text.splitlines()
|
||
if len(lines) > PREVIEW_TEXT_LINE_LIMIT:
|
||
text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT])
|
||
truncated = True
|
||
line_count = max(1, text.count("\n") + 1) if text else 0
|
||
return text, truncated, line_count
|
||
|
||
|
||
def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]:
|
||
text, truncated, line_count = read_text_sample(path)
|
||
return {
|
||
"preview": {
|
||
"kind": "text",
|
||
"format": format_name,
|
||
"text": text,
|
||
"truncated": truncated,
|
||
"lineCount": line_count,
|
||
}
|
||
}
|
||
|
||
|
||
def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]:
|
||
columns: list[str] = []
|
||
rows: list[dict[str, Any]] = []
|
||
total_rows = 0
|
||
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
|
||
reader = csv.reader(handle, delimiter=delimiter)
|
||
for index, row in enumerate(reader):
|
||
if index == 0:
|
||
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
|
||
continue
|
||
total_rows += 1
|
||
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
||
continue
|
||
width = max(len(columns), len(row))
|
||
row_payload: dict[str, Any] = {}
|
||
for column_index in range(width):
|
||
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
|
||
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
|
||
rows.append(row_payload)
|
||
return {
|
||
"preview": {
|
||
"kind": "table",
|
||
"format": format_name,
|
||
"columns": columns,
|
||
"rows": rows,
|
||
"rowCount": total_rows,
|
||
"truncated": total_rows > len(rows),
|
||
}
|
||
}
|
||
|
||
|
||
def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]:
|
||
columns: list[str] = []
|
||
rows: list[dict[str, Any]] = []
|
||
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
|
||
reader = csv.reader(handle, delimiter=delimiter)
|
||
for index, row in enumerate(reader):
|
||
if index == 0:
|
||
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
|
||
continue
|
||
width = max(len(columns), len(row))
|
||
row_payload: dict[str, Any] = {}
|
||
for column_index in range(width):
|
||
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
|
||
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
|
||
rows.append(row_payload)
|
||
return columns, rows
|
||
|
||
|
||
def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]:
|
||
rows: list[Any] = []
|
||
line_count = 0
|
||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||
for raw_line in handle:
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
line_count += 1
|
||
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
||
continue
|
||
try:
|
||
rows.append(json.loads(line))
|
||
except Exception:
|
||
rows.append({"value": line})
|
||
columns: list[str] = []
|
||
for row in rows:
|
||
if isinstance(row, dict):
|
||
for key in row.keys():
|
||
if key not in columns:
|
||
columns.append(key)
|
||
return {
|
||
"preview": {
|
||
"kind": "table",
|
||
"format": "jsonl",
|
||
"columns": columns,
|
||
"rows": rows,
|
||
"rowCount": line_count,
|
||
"truncated": line_count > len(rows),
|
||
}
|
||
}
|
||
|
||
|
||
def inspect_video_file(path: pathlib.Path) -> dict[str, Any]:
|
||
cv2, _ = require_cv2()
|
||
capture = cv2.VideoCapture(str(path))
|
||
if not capture.isOpened():
|
||
return {}
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
|
||
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
|
||
payload = clean_metadata(
|
||
{
|
||
"frameRate": round(fps, 3) if fps > 0 else None,
|
||
"frameCount": frame_count or None,
|
||
"resolution": f"{width}x{height}" if width > 0 and height > 0 else None,
|
||
"duration": round(duration, 3) if duration > 0 else None,
|
||
}
|
||
)
|
||
return payload or {}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]:
|
||
payload: dict[str, Any] = {
|
||
"preview": {
|
||
"kind": "image",
|
||
"format": path.suffix.lower().lstrip(".") or "image",
|
||
}
|
||
}
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
||
payload["preview"]["url"] = data_url_for_path(path, mime_type)
|
||
return payload
|
||
|
||
|
||
def preview_media_payload(path: pathlib.Path, mime_type: str, kind: str) -> dict[str, Any]:
|
||
preview: dict[str, Any] = {
|
||
"kind": kind,
|
||
"format": path.suffix.lower().lstrip(".") or kind,
|
||
}
|
||
if kind == "video":
|
||
preview.update(inspect_video_file(path))
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
||
preview["url"] = data_url_for_path(path, mime_type)
|
||
return {"preview": preview}
|
||
|
||
|
||
def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]:
|
||
try:
|
||
relative = candidate.relative_to(root).as_posix()
|
||
except ValueError:
|
||
relative = candidate.name
|
||
items.append({"path": relative, "sizeBytes": candidate.stat().st_size})
|
||
return items
|
||
|
||
|
||
def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]:
|
||
files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()]
|
||
extensions: dict[str, int] = {}
|
||
total_bytes = 0
|
||
for candidate in files:
|
||
extension = candidate.suffix.lower() or "<none>"
|
||
extensions[extension] = extensions.get(extension, 0) + 1
|
||
total_bytes += candidate.stat().st_size
|
||
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
|
||
return {
|
||
"root": str(root),
|
||
"fileCount": len(files),
|
||
"totalBytes": total_bytes,
|
||
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
|
||
"files": sample_file_entries(files, root),
|
||
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
|
||
}
|
||
|
||
|
||
def optional_json_file(path: pathlib.Path) -> Any:
|
||
if not path.exists():
|
||
return None
|
||
try:
|
||
with path.open("r", encoding="utf-8") as handle:
|
||
return json.load(handle)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def optional_jsonl_rows(path: pathlib.Path) -> list[Any]:
|
||
rows: list[Any] = []
|
||
if not path.exists():
|
||
return rows
|
||
try:
|
||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||
for raw_line in handle:
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
rows.append(json.loads(line))
|
||
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
||
break
|
||
except Exception:
|
||
return []
|
||
return rows
|
||
|
||
|
||
def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None:
|
||
candidates = [path] if path.is_dir() else [path.parent]
|
||
candidates.extend(list((path if path.is_dir() else path.parent).parents))
|
||
for candidate in candidates:
|
||
meta_dir = candidate / "meta"
|
||
if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")):
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]:
|
||
tree = summarize_directory_tree(root)
|
||
meta_dir = root / "meta"
|
||
info = optional_json_file(meta_dir / "info.json")
|
||
stats = optional_json_file(meta_dir / "stats.json")
|
||
tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl")
|
||
episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl")
|
||
parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()]
|
||
video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
|
||
return {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": "lerobot",
|
||
"root": str(root),
|
||
"summary": tree,
|
||
"info": info,
|
||
"stats": stats,
|
||
"tasks": tasks,
|
||
"episodes": episodes,
|
||
"parquetFiles": sample_file_entries(parquet_files, root),
|
||
"parquetCount": len(parquet_files),
|
||
"videoCount": len(video_files),
|
||
}
|
||
}
|
||
|
||
|
||
def sqlite_table_names(connection: sqlite3.Connection) -> list[str]:
|
||
rows = connection.execute(
|
||
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
||
).fetchall()
|
||
return [str(row[0]) for row in rows]
|
||
|
||
|
||
def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]:
|
||
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
||
try:
|
||
tables = sqlite_table_names(connection)
|
||
table_summaries: list[dict[str, Any]] = []
|
||
for table_name in tables[:12]:
|
||
count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0]
|
||
table_summaries.append({"name": table_name, "rowCount": int(count)})
|
||
payload: dict[str, Any] = {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": "sqlite",
|
||
"tables": table_summaries,
|
||
"tableCount": len(tables),
|
||
}
|
||
}
|
||
if "topics" in tables and "messages" in tables:
|
||
topic_rows = connection.execute(
|
||
"select id, name, type, serialization_format from topics order by id"
|
||
).fetchall()
|
||
message_counts = {
|
||
int(topic_id): int(count)
|
||
for topic_id, count in connection.execute(
|
||
"select topic_id, count(*) from messages group by topic_id order by topic_id"
|
||
).fetchall()
|
||
}
|
||
payload["preview"]["format"] = "rosbag2"
|
||
payload["preview"]["topics"] = [
|
||
{
|
||
"id": int(topic_id),
|
||
"name": str(name),
|
||
"type": str(topic_type),
|
||
"serialization": str(serialization),
|
||
"messageCount": message_counts.get(int(topic_id), 0),
|
||
}
|
||
for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT]
|
||
]
|
||
payload["preview"]["messageCount"] = int(sum(message_counts.values()))
|
||
return payload
|
||
finally:
|
||
connection.close()
|
||
|
||
|
||
def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]:
|
||
suffix = path.suffix.lower()
|
||
if suffix == ".db3":
|
||
return summarize_sqlite_payload(path)
|
||
header = ""
|
||
with path.open("rb") as handle:
|
||
header = handle.read(32).decode("utf-8", errors="replace")
|
||
format_name = "mcap" if suffix == ".mcap" else "rosbag1"
|
||
return {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": format_name,
|
||
"header": header.strip(),
|
||
"note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。",
|
||
}
|
||
}
|
||
|
||
|
||
def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None:
|
||
direct = params.get("SOURCE_PATH", "").strip()
|
||
if direct and pathlib.Path(direct).exists():
|
||
return pathlib.Path(direct)
|
||
for artifact in inputs:
|
||
payload = artifact.get("data")
|
||
if isinstance(payload, dict):
|
||
candidate = payload.get("sourcePath")
|
||
if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists():
|
||
return pathlib.Path(candidate)
|
||
source_path = artifact.get("path")
|
||
if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists():
|
||
return pathlib.Path(source_path)
|
||
return None
|
||
|
||
|
||
def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None:
|
||
data = artifact.get("data")
|
||
if not isinstance(data, dict):
|
||
return None
|
||
preview = data.get("preview")
|
||
if isinstance(preview, dict):
|
||
return preview
|
||
return None
|
||
|
||
|
||
def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]:
|
||
extensions: dict[str, int] = {}
|
||
files: list[dict[str, Any]] = []
|
||
for artifact in inputs:
|
||
name = str(artifact.get("name", "") or artifact.get("path", "")).strip()
|
||
extension = pathlib.Path(name).suffix.lower() or "<none>"
|
||
extensions[extension] = extensions.get(extension, 0) + 1
|
||
files.append(
|
||
{
|
||
"path": str(artifact.get("path", "")).strip() or name,
|
||
"name": name,
|
||
"size": str(artifact.get("size", "")),
|
||
}
|
||
)
|
||
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
|
||
return {
|
||
"fileCount": len(files),
|
||
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
|
||
"files": files[:PREVIEW_FILE_LIST_LIMIT],
|
||
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
|
||
}
|
||
|
||
|
||
def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str:
|
||
for artifact in inputs:
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
if not raw_path:
|
||
continue
|
||
parts = pathlib.PurePosixPath(raw_path).parts
|
||
for marker in ("meta", "data", "videos"):
|
||
if marker in parts:
|
||
index = parts.index(marker)
|
||
if index > 0:
|
||
return str(pathlib.PurePosixPath(*parts[:index]))
|
||
return ""
|
||
|
||
|
||
def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]:
|
||
info = None
|
||
stats = None
|
||
tasks: list[Any] = []
|
||
episodes: list[Any] = []
|
||
parquet_files: list[dict[str, Any]] = []
|
||
video_count = 0
|
||
for artifact in inputs:
|
||
name = str(artifact.get("name", "")).strip()
|
||
path = str(artifact.get("path", "")).strip()
|
||
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if name == "info.json" and isinstance(data, dict):
|
||
info = data
|
||
elif name == "stats.json" and isinstance(data, dict):
|
||
stats = data
|
||
elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list):
|
||
tasks = preview["rows"]
|
||
elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list):
|
||
episodes = preview["rows"]
|
||
if name.endswith(".parquet"):
|
||
parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))})
|
||
if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video":
|
||
video_count += 1
|
||
return {
|
||
"preview": {
|
||
"kind": "dataset",
|
||
"format": "lerobot",
|
||
"root": infer_lerobot_root_from_inputs(inputs),
|
||
"summary": summarize_input_artifacts(inputs),
|
||
"info": info,
|
||
"stats": stats,
|
||
"tasks": tasks,
|
||
"episodes": episodes,
|
||
"parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT],
|
||
"parquetCount": len(parquet_files),
|
||
"videoCount": video_count,
|
||
}
|
||
}
|
||
|
||
|
||
def parse_float_param(params: dict[str, str], key: str, default: float) -> float:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
try:
|
||
return float(raw)
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def float_value(value: Any, default: float) -> float:
|
||
if value is None:
|
||
return default
|
||
if isinstance(value, str) and not value.strip():
|
||
return default
|
||
try:
|
||
return float(value)
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def parse_int_param(params: dict[str, str], key: str, default: int) -> int:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
try:
|
||
return int(raw)
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def parse_bool_param(params: dict[str, str], key: str, default: bool = False) -> bool:
|
||
raw = params.get(key, "").strip().lower()
|
||
if not raw:
|
||
return default
|
||
return raw in {"1", "true", "yes", "on"}
|
||
|
||
|
||
def first_payload_with_keys(inputs: list[dict[str, Any]], *keys: str) -> dict[str, Any] | None:
|
||
for artifact in inputs:
|
||
payload = load_artifact_payload(artifact)
|
||
if isinstance(payload, dict) and all(key in payload for key in keys):
|
||
return payload
|
||
return None
|
||
|
||
|
||
def common_input_root(paths: list[str]) -> str:
|
||
filtered = [item for item in paths if item]
|
||
if not filtered:
|
||
return ""
|
||
if len(filtered) == 1:
|
||
return str(pathlib.Path(filtered[0]).parent)
|
||
try:
|
||
return os.path.commonpath(filtered)
|
||
except Exception:
|
||
return str(pathlib.Path(filtered[0]).parent)
|
||
|
||
|
||
def artifact_preview_url(artifact: dict[str, Any]) -> str:
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
value = preview.get("url")
|
||
return value if isinstance(value, str) else ""
|
||
|
||
|
||
def source_ref_from_artifact(artifact: dict[str, Any], relative_path: str = "") -> dict[str, Any]:
|
||
metadata = artifact.get("metadata") if isinstance(artifact.get("metadata"), dict) else {}
|
||
return clean_metadata(
|
||
{
|
||
"name": str(artifact.get("name", "")).strip(),
|
||
"path": str(artifact.get("path", "")).strip(),
|
||
"relativePath": relative_path,
|
||
"previewURL": artifact_preview_url(artifact),
|
||
"mimeType": str(metadata.get("mimeType", "")).strip(),
|
||
}
|
||
) or {}
|
||
|
||
|
||
def materialize_artifact_to_path(artifact: dict[str, Any], destination: pathlib.Path) -> bool:
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
if raw_path and pathlib.Path(raw_path).exists():
|
||
shutil.copy2(raw_path, destination)
|
||
return True
|
||
preview_url = artifact_preview_url(artifact)
|
||
decoded = decode_data_url(preview_url) if preview_url else None
|
||
if decoded is not None:
|
||
_, payload = decoded
|
||
destination.write_bytes(payload)
|
||
return True
|
||
payload = artifact.get("data")
|
||
if isinstance(payload, dict) and destination.suffix.lower() == ".json":
|
||
destination.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return True
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if preview.get("kind") == "text":
|
||
destination.write_text(str(preview.get("text", "")), encoding="utf-8")
|
||
return True
|
||
return False
|
||
|
||
|
||
def upload_relative_path(artifact: dict[str, Any], input_root: str, fallback_name: str) -> str:
|
||
explicit_relative_path = str(artifact.get("relativePath", "")).strip()
|
||
if explicit_relative_path:
|
||
return pathlib.PurePosixPath(explicit_relative_path).as_posix().lstrip("/")
|
||
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
payload_relative_path = str(payload.get("relativePath", "")).strip()
|
||
if payload_relative_path:
|
||
return pathlib.PurePosixPath(payload_relative_path).as_posix().lstrip("/")
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
relative_path = str(fallback_name or artifact.get("name", "")).strip() or "artifact"
|
||
if raw_path and input_root:
|
||
try:
|
||
relative_path = pathlib.Path(raw_path).relative_to(input_root).as_posix()
|
||
except Exception:
|
||
relative_path = pathlib.Path(raw_path).name or relative_path
|
||
if not relative_path:
|
||
relative_path = pathlib.Path(fallback_name or "artifact").name or "artifact"
|
||
return pathlib.PurePosixPath(relative_path).as_posix().lstrip("/")
|
||
|
||
|
||
def materialized_upload_source(artifact: dict[str, Any], staging_dir: pathlib.Path, relative_path: str) -> pathlib.Path | None:
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
if raw_path and os.path.exists(raw_path):
|
||
return pathlib.Path(raw_path)
|
||
destination = staging_dir / "_materialized_upload_inputs" / relative_path
|
||
if destination.exists():
|
||
return destination
|
||
if materialize_artifact_to_path(artifact, destination):
|
||
return destination
|
||
return None
|
||
|
||
|
||
def package_config_payload(item: dict[str, Any], manifest_payload: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"collectionId": item.get("collectionId", ""),
|
||
"meta": item.get("meta", {}),
|
||
"segmentDurationSeconds": manifest_payload.get("segmentDurationSeconds"),
|
||
"sourceVideo": item.get("sourceVideo", ""),
|
||
"startSecond": item.get("startSecond"),
|
||
"endSecond": item.get("endSecond"),
|
||
"duration": item.get("duration") or item.get("clipDuration"),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
}
|
||
|
||
|
||
def package_related_artifact_names(manifest_payload: dict[str, Any]) -> set[str]:
|
||
names: set[str] = set()
|
||
for item in manifest_payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
clip_id = str(item.get("clipId", "")).strip()
|
||
clip_name = str(item.get("clipName", "")).strip()
|
||
annotation_name = str(item.get("annotationFile", "")).strip() or (f"{clip_id}_annotation.json" if clip_id else "")
|
||
quality_name = f"{clip_id}_quality.json" if clip_id else ""
|
||
config_name = f"{clip_id}_config.json" if clip_id else ""
|
||
for candidate in (clip_name, annotation_name, quality_name, config_name):
|
||
if candidate:
|
||
names.add(candidate)
|
||
return names
|
||
|
||
|
||
def materialize_package_artifacts_from_manifest(
|
||
run_id: str,
|
||
node_id: str,
|
||
staging_dir: pathlib.Path,
|
||
manifest_payload: dict[str, Any],
|
||
node_data: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
archive_sources = manifest_payload.get("archiveSources", {}) if isinstance(manifest_payload.get("archiveSources"), dict) else {}
|
||
binding = output_binding(node_data)
|
||
synthetic_artifacts: list[dict[str, Any]] = []
|
||
materialized_root = staging_dir / "_materialized_upload_inputs"
|
||
for item in manifest_payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
clip_id = str(item.get("clipId", "")).strip()
|
||
if not clip_id:
|
||
continue
|
||
package_name = str(item.get("packageName", "")).strip() or f"{clip_id}_package"
|
||
clip_name = str(item.get("clipName", "")).strip() or f"{clip_id}.mp4"
|
||
package_relative_dir = pathlib.PurePosixPath("packages") / package_name
|
||
package_dir = materialized_root / package_relative_dir
|
||
package_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
clip_destination = package_dir / clip_name
|
||
direct_clip_path = str(item.get("clipPath", "")).strip()
|
||
clip_source: pathlib.Path | None = pathlib.Path(direct_clip_path) if direct_clip_path and pathlib.Path(direct_clip_path).exists() else None
|
||
if clip_source is None:
|
||
clip_source = materialize_clip_from_source(item, staging_dir, archive_sources)
|
||
if clip_source is not None and clip_source.exists():
|
||
clip_size = clip_source.stat().st_size
|
||
clip_mime_type = mimetypes.guess_type(clip_name)[0] or "video/mp4"
|
||
clip_artifact = {
|
||
"id": f"{run_id}-{node_id}-{package_name}-{clip_name}",
|
||
"runId": run_id,
|
||
"nodeId": node_id,
|
||
"name": clip_name,
|
||
"type": "video",
|
||
"size": format_size(clip_size),
|
||
"path": str(clip_source),
|
||
"metadata": {"mimeType": clip_mime_type},
|
||
"data": {
|
||
"relativePath": (package_relative_dir / clip_name).as_posix(),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
},
|
||
}
|
||
clip_artifact["relativePath"] = (package_relative_dir / clip_name).as_posix()
|
||
synthetic_artifacts.append(clip_artifact)
|
||
|
||
annotation_name = str(item.get("annotationFile", "")).strip() or f"{clip_id}_annotation.json"
|
||
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
|
||
annotation_artifact = write_json_artifact(run_id, node_id, package_dir, annotation_name, annotation_payload, output=binding)
|
||
annotation_artifact["relativePath"] = (package_relative_dir / annotation_name).as_posix()
|
||
synthetic_artifacts.append(annotation_artifact)
|
||
|
||
quality_name = f"{clip_id}_quality.json"
|
||
quality_payload = {
|
||
"clip_id": clip_id,
|
||
"source_video": item.get("sourceVideo", ""),
|
||
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
|
||
}
|
||
quality_artifact = write_json_artifact(run_id, node_id, package_dir, quality_name, quality_payload, output=binding)
|
||
quality_artifact["relativePath"] = (package_relative_dir / quality_name).as_posix()
|
||
synthetic_artifacts.append(quality_artifact)
|
||
|
||
config_name = f"{clip_id}_config.json"
|
||
config_artifact = write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
package_dir,
|
||
config_name,
|
||
package_config_payload(item, manifest_payload),
|
||
output=binding,
|
||
)
|
||
config_artifact["relativePath"] = (package_relative_dir / config_name).as_posix()
|
||
synthetic_artifacts.append(config_artifact)
|
||
return synthetic_artifacts
|
||
|
||
|
||
def materialize_input_tree(
|
||
inputs: list[dict[str, Any]],
|
||
output_dir: pathlib.Path,
|
||
) -> tuple[pathlib.Path | None, dict[str, dict[str, Any]]]:
|
||
paths = [str(item.get("path", "")).strip() for item in inputs if str(item.get("path", "")).strip()]
|
||
root = common_input_root(paths)
|
||
if not root:
|
||
return None, {}
|
||
target_root = output_dir / "_materialized_inputs"
|
||
if target_root.exists():
|
||
shutil.rmtree(target_root)
|
||
target_root.mkdir(parents=True, exist_ok=True)
|
||
source_refs: dict[str, dict[str, Any]] = {}
|
||
for index, artifact in enumerate(inputs):
|
||
raw_path = str(artifact.get("path", "")).strip()
|
||
relative_path = str(artifact.get("name", "")).strip() or f"artifact-{index}"
|
||
if raw_path:
|
||
try:
|
||
relative_path = pathlib.Path(raw_path).relative_to(root).as_posix()
|
||
except Exception:
|
||
relative_path = pathlib.Path(raw_path).name or relative_path
|
||
destination = target_root / relative_path
|
||
if materialize_artifact_to_path(artifact, destination):
|
||
source_refs[relative_path] = source_ref_from_artifact(artifact, relative_path)
|
||
if destination.is_file() and is_archive_path(destination):
|
||
extracted_files = safe_extract_archive(destination, target_root)
|
||
archive_preview_url = artifact_preview_url(artifact)
|
||
archive_key = artifact_slug(hashlib.sha256(archive_preview_url.encode("utf-8")).hexdigest()[:16]) if archive_preview_url else artifact_slug(relative_path)
|
||
for extracted_relative_path in extracted_files:
|
||
normalized_relative_path = pathlib.PurePosixPath(extracted_relative_path).as_posix().lstrip("/")
|
||
source_refs[normalized_relative_path] = clean_metadata(
|
||
{
|
||
"name": pathlib.PurePosixPath(normalized_relative_path).name,
|
||
"path": str(target_root / normalized_relative_path),
|
||
"relativePath": normalized_relative_path,
|
||
"archiveKey": archive_key,
|
||
"archivePreviewURL": archive_preview_url,
|
||
"archiveMemberPath": normalized_relative_path,
|
||
"archiveName": destination.name,
|
||
"mimeType": mimetypes.guess_type(normalized_relative_path)[0] or "",
|
||
}
|
||
) or {}
|
||
return target_root, source_refs
|
||
|
||
|
||
def detect_xplatform_collections(root: pathlib.Path) -> list[pathlib.Path]:
|
||
if not root.exists():
|
||
return []
|
||
candidates: list[pathlib.Path] = []
|
||
if (root / "meta.json").exists():
|
||
return [root]
|
||
for child in sorted(root.iterdir()):
|
||
if not child.is_dir():
|
||
continue
|
||
if (child / "meta.json").exists():
|
||
candidates.append(child)
|
||
continue
|
||
if any(candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES for candidate in child.rglob("*")):
|
||
candidates.append(child)
|
||
return candidates
|
||
|
||
|
||
def normalize_collection_relative_path(value: pathlib.PurePath | str) -> str:
|
||
text = value.as_posix() if isinstance(value, pathlib.PurePath) else str(value or "")
|
||
normalized = text.strip().strip("/")
|
||
if normalized in {"", "."}:
|
||
return ""
|
||
return normalized
|
||
|
||
|
||
def collection_name_from_source_path(source_path: str, fallback_name: str = "collection") -> str:
|
||
candidate = pathlib.PurePosixPath(str(source_path or "").rstrip("/")).name
|
||
return candidate or fallback_name
|
||
|
||
|
||
def parse_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
|
||
rows: dict[str, dict[str, Any]] = {}
|
||
if not isinstance(value, list):
|
||
return rows
|
||
for item in value:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
filename = str(item.get("filename", "")).strip()
|
||
if not filename:
|
||
continue
|
||
rows[filename] = item
|
||
return rows
|
||
|
||
|
||
def collection_file_entry(
|
||
root: pathlib.Path,
|
||
collection_dir: pathlib.Path,
|
||
candidate: pathlib.Path,
|
||
collection_id: str,
|
||
meta: dict[str, Any],
|
||
video_meta_rows: dict[str, dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
relative_path = candidate.relative_to(root).as_posix()
|
||
collection_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
|
||
collection_relative_file_path = candidate.relative_to(collection_dir).as_posix()
|
||
mime_type = mimetypes.guess_type(candidate.name)[0] or ""
|
||
entry: dict[str, Any] = {
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_dir.name,
|
||
"collectionPath": str(collection_dir),
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"relativePath": relative_path,
|
||
"path": str(candidate),
|
||
"sizeBytes": candidate.stat().st_size,
|
||
"mimeType": mime_type,
|
||
"source": {"path": str(candidate), "relativePath": relative_path, "mimeType": mime_type},
|
||
}
|
||
if candidate.suffix.lower() == ".json":
|
||
payload = optional_json_file(candidate)
|
||
if payload is not None:
|
||
entry["data"] = payload
|
||
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
matched_meta = video_meta_rows.get(candidate.name, {})
|
||
entry.update(
|
||
{
|
||
"videoFolder": collection_relative_file_path.split("/", 1)[0] if "/" in collection_relative_file_path else "",
|
||
"filename": candidate.name,
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
}
|
||
)
|
||
return entry
|
||
|
||
|
||
def collection_summary(
|
||
root: pathlib.Path,
|
||
collection_dir: pathlib.Path,
|
||
meta: dict[str, Any],
|
||
video_meta_rows: dict[str, dict[str, Any]],
|
||
) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:
|
||
files = [candidate for candidate in sorted(collection_dir.rglob("*")) if candidate.is_file()]
|
||
videos = [candidate for candidate in files if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
|
||
total_bytes = sum(candidate.stat().st_size for candidate in files)
|
||
required = ["meta.json", "video_meta.json"]
|
||
missing_required = []
|
||
if not (collection_dir / "meta.json").exists():
|
||
missing_required.append("meta.json")
|
||
if not any((collection_dir / candidate).exists() for candidate in XPLATFORM_VIDEO_META_FILES):
|
||
missing_required.append("video_meta.json")
|
||
has_intrinsics = (collection_dir / "intrinsics.json").exists()
|
||
collection_root_relative_path = normalize_collection_relative_path(collection_dir.relative_to(root))
|
||
collection_id = collection_key(collection_root_relative_path, collection_dir.name)
|
||
items: list[dict[str, Any]] = []
|
||
entries: list[dict[str, Any]] = []
|
||
for candidate in files:
|
||
entry = collection_file_entry(root, collection_dir, candidate, collection_id, meta, video_meta_rows)
|
||
entries.append(entry)
|
||
if candidate.suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
items.append(dict(entry))
|
||
summary = {
|
||
"id": collection_id,
|
||
"name": collection_dir.name,
|
||
"path": str(collection_dir),
|
||
"relativePath": collection_root_relative_path,
|
||
"meta": meta,
|
||
"hasIntrinsics": has_intrinsics,
|
||
"hasVideoMeta": bool(video_meta_rows),
|
||
"requiredFiles": required,
|
||
"missingRequiredFiles": missing_required,
|
||
"videoCount": len(videos),
|
||
"fileCount": len(files),
|
||
"totalBytes": total_bytes,
|
||
"valid": len(missing_required) == 0 and len(videos) > 0,
|
||
}
|
||
return summary, items, entries
|
||
|
||
|
||
def source_path_join(root: str, relative_path: str) -> str:
|
||
if not root:
|
||
return relative_path
|
||
return root.rstrip("/") + "/" + relative_path.lstrip("/")
|
||
|
||
|
||
def manifest_source_ref(entry: dict[str, Any], relative_path: str, source_path: str) -> dict[str, Any]:
|
||
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
|
||
storage_path = str(source.get("path", "") or entry.get("storagePath", "") or source_path_join(source_path, relative_path)).strip()
|
||
return clean_metadata(
|
||
{
|
||
"path": storage_path,
|
||
"relativePath": relative_path,
|
||
"previewURL": str(source.get("previewURL", "")).strip(),
|
||
"mimeType": str(entry.get("mimeType", "")).strip(),
|
||
}
|
||
) or {}
|
||
|
||
|
||
def manifest_entry_json_payload(entry: Any, source_path: str = "") -> Any:
|
||
if not isinstance(entry, dict):
|
||
return None
|
||
inline_data = entry.get("data")
|
||
if inline_data is not None:
|
||
return inline_data
|
||
source = entry.get("source") if isinstance(entry.get("source"), dict) else {}
|
||
candidates = [str(source.get("path", "")).strip(), str(entry.get("storagePath", "")).strip(), str(entry.get("path", "")).strip()]
|
||
for raw_candidate in candidates:
|
||
if not raw_candidate:
|
||
continue
|
||
if "://" in raw_candidate and not raw_candidate.startswith("file://"):
|
||
continue
|
||
candidate_text = raw_candidate[len("file://") :] if raw_candidate.startswith("file://") else raw_candidate
|
||
candidate_path = pathlib.Path(candidate_text)
|
||
if not candidate_path.is_absolute():
|
||
normalized_source_path = str(source_path or "").strip()
|
||
if normalized_source_path and "://" not in normalized_source_path:
|
||
candidate_path = pathlib.Path(normalized_source_path) / candidate_text
|
||
if candidate_path.is_file():
|
||
return optional_json_file(candidate_path)
|
||
return None
|
||
|
||
|
||
def manifest_video_meta_rows(value: Any) -> dict[str, dict[str, Any]]:
|
||
if isinstance(value, dict) and isinstance(value.get("items"), list):
|
||
return parse_video_meta_rows(value.get("items"))
|
||
return parse_video_meta_rows(value)
|
||
|
||
|
||
def collection_key(relative_path: str, fallback_name: str = "collection") -> str:
|
||
normalized = relative_path.strip().strip("/")
|
||
if not normalized:
|
||
return artifact_slug(fallback_name)
|
||
return artifact_slug(normalized)
|
||
|
||
|
||
def fallback_manifest_collections(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
|
||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||
for entry in entries:
|
||
relative_path = str(entry.get("path", "")).strip().lstrip("/")
|
||
if not relative_path:
|
||
continue
|
||
parts = pathlib.PurePosixPath(relative_path).parts
|
||
if len(parts) < 2:
|
||
continue
|
||
grouped.setdefault(parts[0], []).append({**entry, "path": relative_path})
|
||
|
||
collection_summaries: list[dict[str, Any]] = []
|
||
items: list[dict[str, Any]] = []
|
||
indexed_entries: list[dict[str, Any]] = []
|
||
for collection_relative_path in sorted(grouped.keys()):
|
||
collection_entries = grouped[collection_relative_path]
|
||
collection_name = pathlib.PurePosixPath(collection_relative_path).name
|
||
collection_id = collection_key(collection_relative_path, collection_name)
|
||
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/meta.json"), None)
|
||
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == f"{collection_relative_path}/intrinsics.json"), None)
|
||
video_meta_entry = next(
|
||
(
|
||
entry
|
||
for entry in collection_entries
|
||
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
|
||
),
|
||
None,
|
||
)
|
||
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
|
||
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
|
||
meta = meta_payload if isinstance(meta_payload, dict) else {}
|
||
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
|
||
videos = [
|
||
entry
|
||
for entry in collection_entries
|
||
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
|
||
]
|
||
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
|
||
required = ["meta.json", "video_meta.json"]
|
||
missing_required = []
|
||
if meta_entry is None:
|
||
missing_required.append("meta.json")
|
||
if video_meta_entry is None:
|
||
missing_required.append("video_meta.json")
|
||
collection_path = source_path_join(source_path, collection_relative_path)
|
||
collection_summaries.append(
|
||
{
|
||
"id": collection_id,
|
||
"name": collection_name,
|
||
"path": collection_path,
|
||
"relativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"hasIntrinsics": intrinsics_entry is not None,
|
||
"hasVideoMeta": bool(video_meta_rows),
|
||
"requiredFiles": required,
|
||
"missingRequiredFiles": missing_required,
|
||
"videoCount": len(videos),
|
||
"fileCount": len(collection_entries),
|
||
"totalBytes": total_bytes,
|
||
"valid": len(missing_required) == 0 and len(videos) > 0,
|
||
}
|
||
)
|
||
for collection_entry in collection_entries:
|
||
relative_path = str(collection_entry.get("path", "")).strip()
|
||
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
|
||
indexed_entry = {
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
|
||
"mimeType": entry_mime_type,
|
||
"source": manifest_source_ref(collection_entry, relative_path, source_path),
|
||
}
|
||
if "data" in collection_entry:
|
||
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
|
||
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
indexed_entry.update(
|
||
{
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
}
|
||
)
|
||
indexed_entries.append(indexed_entry)
|
||
for video_entry in videos:
|
||
relative_path = str(video_entry.get("path", "")).strip()
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
items.append(
|
||
{
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
"source": manifest_source_ref(video_entry, relative_path, source_path),
|
||
}
|
||
)
|
||
return {
|
||
"datasetType": "x-platform",
|
||
"rootPath": source_path,
|
||
"collectionCount": len(collection_summaries),
|
||
"videoCount": len(items),
|
||
"collections": collection_summaries,
|
||
"items": items,
|
||
"entries": indexed_entries,
|
||
}
|
||
|
||
|
||
def scan_xplatform_manifest_entries(entries: list[dict[str, Any]], source_path: str) -> dict[str, Any]:
|
||
normalized_entries: list[dict[str, Any]] = []
|
||
meta_roots: set[str] = set()
|
||
auxiliary_roots: set[str] = set()
|
||
has_root_metadata = False
|
||
for entry in entries:
|
||
relative_path = str(entry.get("path", "")).strip().lstrip("/")
|
||
if not relative_path:
|
||
continue
|
||
normalized_entry = {**entry, "path": relative_path}
|
||
normalized_entries.append(normalized_entry)
|
||
path_obj = pathlib.PurePosixPath(relative_path)
|
||
parent = path_obj.parent.as_posix()
|
||
if parent in {".", ""}:
|
||
parent = ""
|
||
file_name = path_obj.name.lower()
|
||
if file_name == "meta.json":
|
||
if parent:
|
||
meta_roots.add(parent)
|
||
else:
|
||
has_root_metadata = True
|
||
elif is_xplatform_video_meta_name(file_name) or file_name == "intrinsics.json":
|
||
if parent:
|
||
auxiliary_roots.add(parent)
|
||
else:
|
||
has_root_metadata = True
|
||
|
||
root_candidates = meta_roots or auxiliary_roots
|
||
if has_root_metadata:
|
||
root_candidates = set(root_candidates)
|
||
root_candidates.add("")
|
||
if not root_candidates:
|
||
return fallback_manifest_collections(normalized_entries, source_path)
|
||
|
||
ordered_roots = sorted(root_candidates, key=lambda value: (-len(pathlib.PurePosixPath(value).parts), value))
|
||
grouped: dict[str, list[dict[str, Any]]] = {root: [] for root in ordered_roots}
|
||
unmatched_entries: list[dict[str, Any]] = []
|
||
for entry in normalized_entries:
|
||
relative_path = str(entry.get("path", "")).strip()
|
||
matched_root = next(
|
||
(
|
||
root
|
||
for root in ordered_roots
|
||
if (
|
||
(not root)
|
||
or relative_path == root
|
||
or relative_path.startswith(root + "/")
|
||
)
|
||
),
|
||
None,
|
||
)
|
||
if matched_root is not None:
|
||
grouped.setdefault(matched_root, []).append(entry)
|
||
else:
|
||
unmatched_entries.append(entry)
|
||
|
||
collection_summaries: list[dict[str, Any]] = []
|
||
items: list[dict[str, Any]] = []
|
||
indexed_entries: list[dict[str, Any]] = []
|
||
for collection_relative_path in sorted(grouped.keys()):
|
||
collection_entries = grouped[collection_relative_path]
|
||
if not collection_entries:
|
||
continue
|
||
collection_name = (
|
||
pathlib.PurePosixPath(collection_relative_path).name
|
||
if collection_relative_path
|
||
else collection_name_from_source_path(source_path)
|
||
)
|
||
collection_id = collection_key(collection_relative_path, collection_name)
|
||
meta_path = f"{collection_relative_path}/meta.json" if collection_relative_path else "meta.json"
|
||
intrinsics_path = f"{collection_relative_path}/intrinsics.json" if collection_relative_path else "intrinsics.json"
|
||
meta_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == meta_path), None)
|
||
intrinsics_entry = next((entry for entry in collection_entries if str(entry.get("path", "")).strip() == intrinsics_path), None)
|
||
video_meta_entry = next(
|
||
(
|
||
entry
|
||
for entry in collection_entries
|
||
if is_xplatform_video_meta_name(pathlib.PurePosixPath(str(entry.get("path", "")).strip()).name)
|
||
),
|
||
None,
|
||
)
|
||
meta_payload = manifest_entry_json_payload(meta_entry, source_path)
|
||
video_meta_payload = manifest_entry_json_payload(video_meta_entry, source_path)
|
||
meta = meta_payload if isinstance(meta_payload, dict) else {}
|
||
video_meta_rows = manifest_video_meta_rows(video_meta_payload)
|
||
videos = [
|
||
entry
|
||
for entry in collection_entries
|
||
if pathlib.PurePosixPath(str(entry.get("path", ""))).suffix.lower() in VIDEO_FILE_SUFFIXES
|
||
]
|
||
total_bytes = sum(int(entry.get("sizeBytes", 0) or 0) for entry in collection_entries)
|
||
required = ["meta.json", "video_meta.json"]
|
||
missing_required = []
|
||
if meta_entry is None:
|
||
missing_required.append("meta.json")
|
||
if video_meta_entry is None:
|
||
missing_required.append("video_meta.json")
|
||
collection_path = source_path if not collection_relative_path else source_path_join(source_path, collection_relative_path)
|
||
collection_summaries.append(
|
||
{
|
||
"id": collection_id,
|
||
"name": collection_name,
|
||
"path": collection_path,
|
||
"relativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"hasIntrinsics": intrinsics_entry is not None,
|
||
"hasVideoMeta": bool(video_meta_rows),
|
||
"requiredFiles": required,
|
||
"missingRequiredFiles": missing_required,
|
||
"videoCount": len(videos),
|
||
"fileCount": len(collection_entries),
|
||
"totalBytes": total_bytes,
|
||
"valid": len(missing_required) == 0 and len(videos) > 0,
|
||
}
|
||
)
|
||
for collection_entry in collection_entries:
|
||
relative_path = str(collection_entry.get("path", "")).strip()
|
||
entry_mime_type = str(collection_entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
|
||
indexed_entry = {
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(collection_entry.get("sizeBytes", 0) or 0),
|
||
"mimeType": entry_mime_type,
|
||
"source": manifest_source_ref(collection_entry, relative_path, source_path),
|
||
}
|
||
if "data" in collection_entry:
|
||
indexed_entry["data"] = normalize_json_payload(collection_entry.get("data"))
|
||
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
indexed_entry.update(
|
||
{
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
}
|
||
)
|
||
indexed_entries.append(indexed_entry)
|
||
for video_entry in videos:
|
||
relative_path = str(video_entry.get("path", "")).strip()
|
||
collection_relative_video_path = str(pathlib.PurePosixPath(relative_path).relative_to(collection_relative_path))
|
||
matched_meta = video_meta_rows.get(pathlib.PurePosixPath(relative_path).name, {})
|
||
items.append(
|
||
{
|
||
"collectionId": collection_id,
|
||
"collectionName": collection_name,
|
||
"collectionPath": collection_path,
|
||
"collectionRelativePath": collection_relative_path,
|
||
"meta": meta,
|
||
"videoFolder": collection_relative_video_path.split("/", 1)[0] if "/" in collection_relative_video_path else "",
|
||
"filename": pathlib.PurePosixPath(relative_path).name,
|
||
"relativePath": relative_path,
|
||
"path": source_path_join(source_path, relative_path),
|
||
"sizeBytes": int(video_entry.get("sizeBytes", 0) or 0),
|
||
"codec": matched_meta.get("codec"),
|
||
"resolution": matched_meta.get("resolution"),
|
||
"frameRate": matched_meta.get("frameRate"),
|
||
"duration": matched_meta.get("duration"),
|
||
"startTime": matched_meta.get("startTime"),
|
||
"source": manifest_source_ref(video_entry, relative_path, source_path),
|
||
}
|
||
)
|
||
if unmatched_entries:
|
||
fallback_payload = fallback_manifest_collections(unmatched_entries, source_path)
|
||
collection_summaries.extend(fallback_payload.get("collections", []))
|
||
items.extend(fallback_payload.get("items", []))
|
||
indexed_entries.extend(fallback_payload.get("entries", []))
|
||
return {
|
||
"datasetType": "x-platform",
|
||
"rootPath": source_path,
|
||
"collectionCount": len(collection_summaries),
|
||
"videoCount": len(items),
|
||
"collections": collection_summaries,
|
||
"items": items,
|
||
"entries": indexed_entries,
|
||
}
|
||
|
||
|
||
def with_xplatform_source_context(payload: dict[str, Any], source_manifest: dict[str, Any]) -> dict[str, Any]:
|
||
for key in (
|
||
"dataSource",
|
||
"sourcePath",
|
||
"scanMode",
|
||
"listedCount",
|
||
"newCount",
|
||
"sourceFingerprint",
|
||
"bucket",
|
||
"prefix",
|
||
"pattern",
|
||
"endpoint",
|
||
):
|
||
if key in source_manifest:
|
||
payload[key] = source_manifest.get(key)
|
||
return payload
|
||
|
||
|
||
def attach_xplatform_catalog_payload(payload: dict[str, Any], catalog_payload: dict[str, Any]) -> dict[str, Any]:
|
||
payload["catalogCollectionCount"] = int(catalog_payload.get("collectionCount", 0) or 0)
|
||
payload["catalogVideoCount"] = int(catalog_payload.get("videoCount", 0) or 0)
|
||
payload["catalogCollections"] = catalog_payload.get("collections", [])
|
||
payload["catalogItems"] = catalog_payload.get("items", [])
|
||
payload["catalogEntries"] = catalog_payload.get("entries", [])
|
||
return payload
|
||
|
||
|
||
def apply_source_refs_to_items(
|
||
items: list[dict[str, Any]],
|
||
source_refs: dict[str, dict[str, Any]],
|
||
) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]:
|
||
if not source_refs:
|
||
return items, {}
|
||
archive_sources: dict[str, dict[str, Any]] = {}
|
||
for item in items:
|
||
relative_path = str(item.get("relativePath", "")).strip()
|
||
if not relative_path:
|
||
continue
|
||
source_ref = source_refs.get(relative_path)
|
||
if source_ref:
|
||
archive_key = str(source_ref.get("archiveKey", "")).strip()
|
||
archive_preview_url = str(source_ref.get("archivePreviewURL", "")).strip()
|
||
if archive_key and archive_preview_url and archive_key not in archive_sources:
|
||
archive_sources[archive_key] = clean_metadata(
|
||
{
|
||
"name": str(source_ref.get("archiveName", "")).strip() or "archive",
|
||
"previewURL": archive_preview_url,
|
||
}
|
||
) or {}
|
||
item_source = {**source_ref, "path": item.get("path", "")}
|
||
item_source.pop("archivePreviewURL", None)
|
||
item_source.pop("archiveName", None)
|
||
item["source"] = item_source
|
||
return items, archive_sources
|
||
|
||
|
||
def scan_xplatform_dataset(root: pathlib.Path, source_refs: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]:
|
||
collections = detect_xplatform_collections(root)
|
||
collection_summaries: list[dict[str, Any]] = []
|
||
items: list[dict[str, Any]] = []
|
||
entries: list[dict[str, Any]] = []
|
||
for collection_dir in collections:
|
||
meta = optional_json_file(collection_dir / "meta.json")
|
||
meta = meta if isinstance(meta, dict) else {}
|
||
video_meta_rows, _ = load_collection_video_meta(collection_dir)
|
||
summary, collection_items, collection_entries = collection_summary(root, collection_dir, meta, video_meta_rows)
|
||
collection_summaries.append(summary)
|
||
items.extend(collection_items)
|
||
entries.extend(collection_entries)
|
||
items, archive_sources = apply_source_refs_to_items(items, source_refs or {})
|
||
entries, entry_archive_sources = apply_source_refs_to_items(entries, source_refs or {})
|
||
archive_sources.update(entry_archive_sources)
|
||
payload = {
|
||
"datasetType": "x-platform",
|
||
"rootPath": str(root),
|
||
"collectionCount": len(collection_summaries),
|
||
"videoCount": len(items),
|
||
"collections": collection_summaries,
|
||
"items": items,
|
||
"entries": entries,
|
||
}
|
||
if archive_sources:
|
||
payload["archiveSources"] = archive_sources
|
||
return payload
|
||
|
||
|
||
def load_xplatform_index_payload(
|
||
params: dict[str, str],
|
||
inputs: list[dict[str, Any]],
|
||
output_dir: pathlib.Path,
|
||
) -> dict[str, Any]:
|
||
source_manifest = first_payload_with_keys(inputs, "entries")
|
||
if source_manifest is not None:
|
||
entries = source_manifest.get("entries")
|
||
if isinstance(entries, list):
|
||
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
|
||
payload = scan_xplatform_manifest_entries(entries, manifest_source_path)
|
||
catalog_entries = source_manifest.get("catalogEntries")
|
||
if isinstance(catalog_entries, list):
|
||
catalog_payload = scan_xplatform_manifest_entries(catalog_entries, manifest_source_path)
|
||
attach_xplatform_catalog_payload(payload, catalog_payload)
|
||
return with_xplatform_source_context(payload, source_manifest)
|
||
manifest_source_path = str(source_manifest.get("sourcePath", "")).strip()
|
||
if manifest_source_path and pathlib.Path(manifest_source_path).exists():
|
||
root = pathlib.Path(manifest_source_path)
|
||
return scan_xplatform_dataset(root if root.is_dir() else root.parent)
|
||
for artifact in inputs:
|
||
name = str(artifact.get("name", "")).strip().lower()
|
||
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||
if name != "source-manifest.json" and not any(key in data for key in ("entriesCount", "catalogEntriesCount", "sourcePath")):
|
||
continue
|
||
raise RuntimeError("X 平台数据集索引未能读取 source-manifest 的 entries,拒绝回退到空目录扫描")
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
root = source_path if source_path.is_dir() else source_path.parent
|
||
return scan_xplatform_dataset(root)
|
||
materialized_root, source_refs = materialize_input_tree(inputs, output_dir)
|
||
if materialized_root is not None:
|
||
return scan_xplatform_dataset(materialized_root, source_refs)
|
||
raise RuntimeError("X 平台数据集索引需要 SOURCE_PATH 或上游输入提供真实数据目录")
|
||
|
||
|
||
def normalize_date_value(value: Any) -> str:
|
||
text = str(value or "").strip()
|
||
digits = "".join(character for character in text if character.isdigit())
|
||
if len(digits) >= 8:
|
||
return digits[:8]
|
||
return ""
|
||
|
||
|
||
def date_in_range(value: Any, start: str, end: str) -> bool:
|
||
normalized = normalize_date_value(value)
|
||
if not normalized:
|
||
return False
|
||
if start and normalized < start:
|
||
return False
|
||
if end and normalized > end:
|
||
return False
|
||
return True
|
||
|
||
|
||
def materialize_archive_member(
|
||
source: dict[str, Any],
|
||
archive_sources: dict[str, Any],
|
||
default_name: str,
|
||
output_dir: pathlib.Path,
|
||
) -> pathlib.Path | None:
|
||
archive_key = str(source.get("archiveKey", "")).strip()
|
||
archive_source = archive_sources.get(archive_key) if isinstance(archive_sources, dict) else {}
|
||
archive_source = archive_source if isinstance(archive_source, dict) else {}
|
||
preview_url = str(archive_source.get("previewURL", "")).strip()
|
||
member_path = str(source.get("archiveMemberPath", "")).strip()
|
||
decoded = decode_data_url(preview_url) if preview_url and member_path else None
|
||
if decoded is None:
|
||
return None
|
||
_, payload = decoded
|
||
archive_name = pathlib.PurePosixPath(str(archive_source.get("name", "")).strip() or "archive").name
|
||
archive_path = output_dir / "_materialized_sources" / archive_name
|
||
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||
archive_path.write_bytes(payload)
|
||
destination = output_dir / "_materialized_sources" / default_name
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
if archive_name.lower().endswith(".zip"):
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
with archive.open(member_path) as handle:
|
||
destination.write_bytes(handle.read())
|
||
return destination
|
||
with tarfile.open(archive_path, "r:*") as archive:
|
||
extracted = archive.extractfile(member_path)
|
||
if extracted is None:
|
||
return None
|
||
with extracted:
|
||
destination.write_bytes(extracted.read())
|
||
return destination
|
||
|
||
|
||
def source_path_from_item(
|
||
item: dict[str, Any],
|
||
default_name: str,
|
||
output_dir: pathlib.Path,
|
||
archive_sources: dict[str, Any] | None = None,
|
||
) -> pathlib.Path:
|
||
source = item.get("source") if isinstance(item.get("source"), dict) else {}
|
||
direct = str(source.get("path", "") or item.get("path", "")).strip()
|
||
if direct and pathlib.Path(direct).exists():
|
||
return pathlib.Path(direct)
|
||
if direct.startswith("bos://"):
|
||
cached_target = workspace_cached_bos_source_path(direct, output_dir)
|
||
if cached_target is not None and cached_target.exists() and cached_target.stat().st_size > 0:
|
||
return cached_target
|
||
cached_bos_source = download_bos_source(direct, default_name, output_dir)
|
||
if cached_bos_source is not None:
|
||
return cached_bos_source
|
||
preview_url = str(source.get("previewURL", "")).strip()
|
||
if cached_target is not None and preview_url:
|
||
cached_remote_source = download_remote_source_to_path(preview_url, cached_target)
|
||
if cached_remote_source is not None:
|
||
return cached_remote_source
|
||
archive_member = materialize_archive_member(source, archive_sources or {}, default_name, output_dir)
|
||
if archive_member is not None:
|
||
return archive_member
|
||
preview_url = str(source.get("previewURL", "")).strip()
|
||
remote_path = download_remote_source(preview_url, default_name, output_dir) if preview_url else None
|
||
if remote_path is not None:
|
||
return remote_path
|
||
decoded = decode_data_url(preview_url) if preview_url else None
|
||
if decoded is None:
|
||
raise RuntimeError(f"无法获取真实输入文件: {default_name}")
|
||
_, payload = decoded
|
||
path = output_dir / "_materialized_sources" / default_name
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(payload)
|
||
return path
|
||
|
||
|
||
def clip_segment_bounds(item: dict[str, Any]) -> tuple[float, float]:
|
||
start_second = float(item.get("startSecond") or 0.0)
|
||
end_second = float(item.get("endSecond") or 0.0)
|
||
duration = float(item.get("duration") or 0.0)
|
||
if end_second <= start_second and duration > 0:
|
||
end_second = start_second + duration
|
||
if end_second <= start_second:
|
||
end_second = start_second + 0.1
|
||
return start_second, end_second
|
||
|
||
|
||
def materialize_clip_from_source(
|
||
item: dict[str, Any],
|
||
output_dir: pathlib.Path,
|
||
archive_sources: dict[str, Any] | None = None,
|
||
) -> pathlib.Path | None:
|
||
source = item.get("source") if isinstance(item.get("source"), dict) else {}
|
||
source_name = (
|
||
str(item.get("sourceVideo", "")).strip()
|
||
or pathlib.PurePosixPath(str(source.get("path", "")).strip()).name
|
||
or "source.mp4"
|
||
)
|
||
clip_name = str(item.get("clipName", "")).strip() or f"{pathlib.Path(source_name).stem}_clip.mp4"
|
||
destination = output_dir / "_materialized_clips" / clip_name
|
||
if destination.exists():
|
||
return destination
|
||
start_second, end_second = clip_segment_bounds(item)
|
||
remote_source_url = remote_video_url_for_source(source) if source else ""
|
||
if remote_source_url:
|
||
try:
|
||
write_video_segment_from_remote_url(remote_source_url, destination, start_second, end_second)
|
||
return destination
|
||
except RuntimeError as exc:
|
||
log(f"[operator-runner][warn] fallback to temporary source materialization for {clip_name}: {exc}")
|
||
if not source:
|
||
return None
|
||
source_path = source_path_from_item({"source": source}, source_name, output_dir, archive_sources)
|
||
write_video_segment(source_path, destination, start_second, end_second)
|
||
return destination
|
||
|
||
|
||
def analyze_video_quality_metrics(path: pathlib.Path, clarity_scale: float) -> dict[str, Any]:
|
||
cv2, np = require_cv2()
|
||
capture = cv2.VideoCapture(str(path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频文件: {path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0)
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0)
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0)
|
||
positions = sampled_frame_positions(frame_count)
|
||
if not positions:
|
||
positions = [0]
|
||
clarities: list[float] = []
|
||
shifts: list[tuple[float, float]] = []
|
||
skin_ratios: list[float] = []
|
||
previous_gray = None
|
||
previous_position: int | None = None
|
||
analysis_diagonal = math.sqrt((width or 1) ** 2 + (height or 1) ** 2)
|
||
for position in positions:
|
||
if frame_count > 0:
|
||
capture.set(cv2.CAP_PROP_POS_FRAMES, position)
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
continue
|
||
analysis_frame = resize_quality_frame(frame)
|
||
analysis_height, analysis_width = analysis_frame.shape[:2]
|
||
analysis_diagonal = math.sqrt((analysis_width or 1) ** 2 + (analysis_height or 1) ** 2)
|
||
gray = cv2.cvtColor(analysis_frame, cv2.COLOR_BGR2GRAY)
|
||
laplacian = cv2.Laplacian(gray, cv2.CV_64F).var()
|
||
clarities.append(float(laplacian))
|
||
if previous_gray is not None:
|
||
gap = max(position - previous_position, 1) if previous_position is not None else 1
|
||
shift, _ = cv2.phaseCorrelate(np.float32(previous_gray), np.float32(gray))
|
||
shifts.append((float(shift[0]) / gap, float(shift[1]) / gap))
|
||
mask = build_skin_mask(analysis_frame)
|
||
skin_ratios.append(largest_skin_component_ratio(mask))
|
||
previous_gray = gray
|
||
previous_position = position
|
||
clarity_raw = sum(clarities) / max(len(clarities), 1)
|
||
clarity = 1.0 - math.exp(-max(clarity_raw, 0.0) / max(clarity_scale, 1.0))
|
||
shake = motion_jitter_ratio(shifts, analysis_diagonal) if shifts else 0.0
|
||
hand_occlusion = sum(skin_ratios) / max(len(skin_ratios), 1) if skin_ratios else 0.0
|
||
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
|
||
quality_score = max(0.0, min(1.0, (clarity + (1.0 - min(shake, 1.0)) + (1.0 - min(hand_occlusion, 1.0))) / 3.0))
|
||
return {
|
||
"codec": "opencv",
|
||
"resolution": f"{width}x{height}" if width > 0 and height > 0 else "",
|
||
"frameRate": round(fps, 3) if fps > 0 else None,
|
||
"duration": round(duration, 3) if duration > 0 else None,
|
||
"clarity": round(clarity, 4),
|
||
"shake": round(shake, 4),
|
||
"handOcclusion": round(hand_occlusion, 4),
|
||
"qualityScore": round(quality_score, 4),
|
||
"sampleCount": len(clarities),
|
||
}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def stable_sample_hit(value: str, percent: float) -> bool:
|
||
if percent >= 100:
|
||
return True
|
||
if percent <= 0:
|
||
return False
|
||
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||
score = int(digest[:8], 16) / 0xFFFFFFFF
|
||
return score <= (percent / 100.0)
|
||
|
||
|
||
def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None:
|
||
suffix = path.suffix.lower()
|
||
if suffix in TABULAR_FILE_SUFFIXES:
|
||
return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip("."))
|
||
if suffix == ".jsonl":
|
||
return preview_jsonl_payload(path)
|
||
if suffix in TEXT_FILE_SUFFIXES:
|
||
format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".")
|
||
return preview_text_payload(path, format_name)
|
||
if suffix in IMAGE_FILE_SUFFIXES:
|
||
return preview_image_payload(path, mime_type)
|
||
if suffix in ROSBAG_FILE_SUFFIXES:
|
||
return summarize_rosbag_file(path)
|
||
if suffix == ".pdf":
|
||
return {"preview": {"kind": "document", "format": "pdf"}}
|
||
if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES:
|
||
return preview_media_payload(path, mime_type or "video/mp4", "video")
|
||
if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
|
||
return preview_media_payload(path, mime_type or "audio/mpeg", "audio")
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
||
return {"preview": {"kind": "binary", "format": suffix.lstrip(".") or "file", "url": data_url_for_path(path, mime_type)}}
|
||
return None
|
||
|
||
|
||
def build_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
path: pathlib.Path,
|
||
artifact_type: str,
|
||
data: dict[str, Any] | None = None,
|
||
metadata: dict[str, Any] | None = None,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
mime_type, _ = mimetypes.guess_type(path.name)
|
||
size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8"))
|
||
sha256_value = ""
|
||
sha256_metadata: dict[str, Any] = {}
|
||
if path.exists():
|
||
sha256_value, sha256_metadata = artifact_sha256(path, size_bytes)
|
||
artifact_metadata = clean_metadata(
|
||
{
|
||
**(metadata or {}),
|
||
**sha256_metadata,
|
||
"sha256": sha256_value,
|
||
"mimeType": mime_type or "",
|
||
}
|
||
)
|
||
artifact = {
|
||
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}",
|
||
"runId": run_id,
|
||
"nodeId": node_id,
|
||
"name": path.name,
|
||
"type": artifact_type,
|
||
"size": format_size(size_bytes),
|
||
"path": str(path),
|
||
}
|
||
if output:
|
||
if str(output.get("id", "")).strip():
|
||
artifact["portId"] = str(output.get("id", "")).strip()
|
||
if str(output.get("name", "")).strip():
|
||
artifact["portName"] = str(output.get("name", "")).strip()
|
||
if artifact_metadata:
|
||
artifact["metadata"] = artifact_metadata
|
||
if data:
|
||
artifact["data"] = data
|
||
return artifact
|
||
|
||
|
||
def build_virtual_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
name: str,
|
||
path: str,
|
||
artifact_type: str,
|
||
size_bytes: int,
|
||
data: dict[str, Any] | None = None,
|
||
metadata: dict[str, Any] | None = None,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
artifact = {
|
||
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else name)}-{artifact_slug(name)}",
|
||
"runId": run_id,
|
||
"nodeId": node_id,
|
||
"name": name,
|
||
"type": artifact_type,
|
||
"size": format_size(max(0, size_bytes)),
|
||
"path": path,
|
||
}
|
||
if output:
|
||
if str(output.get("id", "")).strip():
|
||
artifact["portId"] = str(output.get("id", "")).strip()
|
||
if str(output.get("name", "")).strip():
|
||
artifact["portName"] = str(output.get("name", "")).strip()
|
||
if metadata:
|
||
cleaned = clean_metadata(metadata)
|
||
if cleaned:
|
||
artifact["metadata"] = cleaned
|
||
if data:
|
||
artifact["data"] = data
|
||
return artifact
|
||
|
||
|
||
def dataset_partition_keys(payload: dict[str, Any]) -> list[str]:
|
||
keys: list[str] = []
|
||
for key in DATASET_PARTITION_KEYS:
|
||
if isinstance(payload.get(key), list):
|
||
keys.append(key)
|
||
return keys
|
||
|
||
|
||
def should_externalize_structured_payload(payload: dict[str, Any]) -> bool:
|
||
if dataset_partition_keys(payload):
|
||
return True
|
||
dataset_type = str(payload.get("datasetType", "")).strip()
|
||
if dataset_type:
|
||
return True
|
||
return False
|
||
|
||
|
||
def dataset_payload_summary(payload: dict[str, Any]) -> dict[str, Any]:
|
||
summary: dict[str, Any] = {}
|
||
for key in (
|
||
"datasetType",
|
||
"count",
|
||
"itemCount",
|
||
"collectionCount",
|
||
"entryCount",
|
||
"videoCount",
|
||
"clipCount",
|
||
"annotationCount",
|
||
"packageCount",
|
||
"sampledCount",
|
||
"passedCount",
|
||
"failedCount",
|
||
"validCollectionCount",
|
||
"invalidCollectionCount",
|
||
"dataSource",
|
||
"sourcePath",
|
||
"bucket",
|
||
"prefix",
|
||
"endpoint",
|
||
"pattern",
|
||
"scanMode",
|
||
"sourceFingerprint",
|
||
"listedCount",
|
||
"newCount",
|
||
"rootPath",
|
||
"segmentDurationSeconds",
|
||
):
|
||
if key in payload:
|
||
summary[key] = json_compatible(payload[key])
|
||
if "preview" in payload:
|
||
summary["preview"] = json_compatible(payload.get("preview"))
|
||
for key in dataset_partition_keys(payload):
|
||
value = payload.get(key)
|
||
if isinstance(value, list):
|
||
summary[f"{key}Count"] = len(value)
|
||
if "preview" not in summary:
|
||
summary["preview"] = {
|
||
"kind": "dataset",
|
||
"format": str(payload.get("datasetType", "")).strip() or "json",
|
||
"counts": {key: summary[key] for key in summary.keys() if key.endswith("Count") or key == "count"},
|
||
}
|
||
return summary
|
||
|
||
|
||
def chunk_list(values: list[Any], chunk_size: int = DATASET_PARTITION_SIZE) -> list[list[Any]]:
|
||
if chunk_size <= 0:
|
||
chunk_size = DATASET_PARTITION_SIZE
|
||
return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] or [[]]
|
||
|
||
|
||
def externalize_structured_payload(path: pathlib.Path, payload: dict[str, Any]) -> dict[str, Any]:
|
||
normalized_payload = json_compatible(payload)
|
||
dataset_dir = path.parent / "_datasets" / artifact_slug(path.stem or path.name)
|
||
partitions_dir = dataset_dir / "partitions"
|
||
partitions_dir.mkdir(parents=True, exist_ok=True)
|
||
partition_files: dict[str, list[str]] = {}
|
||
partition_count = 0
|
||
partition_keys = dataset_partition_keys(normalized_payload)
|
||
for key in partition_keys:
|
||
values = normalized_payload.get(key)
|
||
if not isinstance(values, list):
|
||
continue
|
||
files: list[str] = []
|
||
for index, chunk in enumerate(chunk_list(values), start=1):
|
||
partition_path = partitions_dir / f"{key}-p{index:06d}.json"
|
||
write_json_file(partition_path, chunk)
|
||
files.append(str(partition_path))
|
||
partition_files[key] = files
|
||
partition_count += len(files)
|
||
manifest = {
|
||
"kind": "dataset",
|
||
"datasetType": str(normalized_payload.get("datasetType", "")).strip() or "json",
|
||
"payloadPath": str(path),
|
||
"partitionDir": str(partitions_dir),
|
||
"partitionKeys": partition_keys,
|
||
"partitionFiles": partition_files,
|
||
"summary": dataset_payload_summary(normalized_payload),
|
||
"generatedAt": now_iso(),
|
||
}
|
||
manifest_path = write_json_file(dataset_dir / "manifest.json", manifest)
|
||
data_ref = {
|
||
"kind": "dataset",
|
||
"datasetType": manifest["datasetType"],
|
||
"manifestPath": str(manifest_path),
|
||
"payloadPath": str(path),
|
||
"partitionDir": str(partitions_dir),
|
||
"partitionCount": partition_count,
|
||
}
|
||
item_count = 0
|
||
for key in partition_keys:
|
||
values = normalized_payload.get(key)
|
||
if isinstance(values, list):
|
||
item_count += len(values)
|
||
if item_count > 0:
|
||
data_ref["itemCount"] = item_count
|
||
summary = dataset_payload_summary(normalized_payload)
|
||
summary["dataRef"] = data_ref
|
||
return summary
|
||
|
||
|
||
def write_json_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
filename: str,
|
||
payload: dict[str, Any],
|
||
artifact_type: str = "json",
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
normalized_payload = json_compatible(payload)
|
||
path = output_dir / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8") as handle:
|
||
json.dump(normalized_payload, handle, ensure_ascii=False, indent=2)
|
||
artifact_data: dict[str, Any] | None = normalized_payload
|
||
if should_externalize_structured_payload(normalized_payload):
|
||
artifact_data = externalize_structured_payload(path, normalized_payload)
|
||
return build_artifact(run_id, node_id, path, artifact_type, data=artifact_data, output=output)
|
||
|
||
|
||
def summarize_json_like(value: Any) -> dict[str, Any]:
|
||
if isinstance(value, dict):
|
||
return {"kind": "object", "keys": sorted(value.keys())}
|
||
if isinstance(value, list):
|
||
return {"kind": "list", "length": len(value)}
|
||
return {"kind": type(value).__name__, "value": value}
|
||
|
||
|
||
def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]:
|
||
data = artifact.get("data")
|
||
if isinstance(data, dict):
|
||
data_ref = data.get("dataRef") if isinstance(data.get("dataRef"), dict) else {}
|
||
payload_path = str(data_ref.get("payloadPath", "")).strip()
|
||
if payload_path:
|
||
payload = load_payload_from_file(pathlib.Path(payload_path))
|
||
if isinstance(payload, dict):
|
||
return payload
|
||
if isinstance(data, dict):
|
||
return data
|
||
path_value = artifact.get("path")
|
||
if isinstance(path_value, str) and path_value:
|
||
payload = load_payload_from_file(pathlib.Path(path_value))
|
||
if payload is not None:
|
||
return payload
|
||
summary = {
|
||
"name": artifact.get("name", ""),
|
||
"type": artifact.get("type", ""),
|
||
"path": artifact.get("path", ""),
|
||
"size": artifact.get("size", ""),
|
||
}
|
||
return {key: value for key, value in summary.items() if value}
|
||
|
||
|
||
def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]:
|
||
merged: dict[str, Any] = {}
|
||
for value in values:
|
||
merged.update(value)
|
||
return merged
|
||
|
||
|
||
def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]:
|
||
entries: list[dict[str, Any]] = []
|
||
if not base_path.exists():
|
||
return entries
|
||
if base_path.is_file():
|
||
relative = base_path.name
|
||
if pattern and not fnmatch.fnmatch(relative, pattern):
|
||
return entries
|
||
return [
|
||
{
|
||
"path": relative,
|
||
"sizeBytes": base_path.stat().st_size,
|
||
"sha256": sha256_file(base_path),
|
||
}
|
||
]
|
||
for candidate in sorted(base_path.rglob("*")):
|
||
if candidate.is_dir():
|
||
continue
|
||
relative = candidate.relative_to(base_path).as_posix()
|
||
if pattern and not fnmatch.fnmatch(relative, pattern):
|
||
continue
|
||
entries.append(
|
||
{
|
||
"path": relative,
|
||
"sizeBytes": candidate.stat().st_size,
|
||
"sha256": sha256_file(candidate),
|
||
}
|
||
)
|
||
return entries
|
||
|
||
|
||
def list_bos_entries(
|
||
bucket: str,
|
||
prefix: str,
|
||
pattern: str,
|
||
endpoint: str,
|
||
access_key_id: str,
|
||
secret_access_key: str,
|
||
) -> list[dict[str, Any]]:
|
||
client, scheme = build_bos_client(endpoint, access_key_id, secret_access_key)
|
||
normalized_prefix = bos_prefix(prefix)
|
||
prefix_with_slash = f"{normalized_prefix}/" if normalized_prefix else ""
|
||
entries: list[dict[str, Any]] = []
|
||
for candidate in client.list_all_objects(bucket, prefix=normalized_prefix or None):
|
||
key = text_value(getattr(candidate, "key", "")).strip().lstrip("/")
|
||
if not key or key.endswith("/"):
|
||
continue
|
||
if normalized_prefix:
|
||
if key == normalized_prefix:
|
||
continue
|
||
if not key.startswith(prefix_with_slash):
|
||
continue
|
||
relative_path = key[len(prefix_with_slash):]
|
||
else:
|
||
relative_path = key
|
||
if not relative_path:
|
||
continue
|
||
if pattern and not fnmatch.fnmatch(relative_path, pattern):
|
||
continue
|
||
size_bytes = int(getattr(candidate, "size", 0) or 0)
|
||
mime_type = mimetypes.guess_type(relative_path)[0] or ""
|
||
storage_path = canonical_bos_path(bucket, key)
|
||
entry: dict[str, Any] = {
|
||
"path": relative_path,
|
||
"storagePath": storage_path,
|
||
"sizeBytes": size_bytes,
|
||
"mimeType": mime_type,
|
||
"etag": text_value(getattr(candidate, "e_tag", "") or getattr(candidate, "etag", "")),
|
||
"versionId": text_value(getattr(candidate, "version_id", "") or getattr(candidate, "versionId", "")),
|
||
"lastModified": text_value(getattr(candidate, "last_modified", "") or getattr(candidate, "lastModified", "")),
|
||
"source": {"path": storage_path},
|
||
}
|
||
file_name = pathlib.PurePosixPath(relative_path).name.lower()
|
||
should_inline_json = relative_path.lower().endswith(".json") and (
|
||
file_name in XPLATFORM_INLINE_JSON_FILES or size_bytes <= PREVIEW_INLINE_BINARY_LIMIT
|
||
)
|
||
if should_inline_json:
|
||
raw = client.get_object_as_string(bucket, key)
|
||
payload_text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
|
||
try:
|
||
entry["data"] = json.loads(payload_text)
|
||
except Exception:
|
||
entry["data"] = {"value": payload_text}
|
||
elif pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
|
||
entry["source"]["previewURL"] = client.generate_pre_signed_url(
|
||
bucket,
|
||
key,
|
||
expiration_in_seconds=3600,
|
||
protocol=scheme,
|
||
)
|
||
entries.append(clean_metadata(entry) or {})
|
||
return entries
|
||
|
||
|
||
def source_entry_fingerprint(data_source: str, bucket: str, entry: dict[str, Any]) -> str:
|
||
normalized_source = (data_source or "BOS").strip().upper() or "BOS"
|
||
relative_path = str(entry.get("path", "")).strip()
|
||
version_id = str(entry.get("versionId", "")).strip()
|
||
sha256 = str(entry.get("sha256", "")).strip()
|
||
etag = str(entry.get("etag", "")).strip()
|
||
size_bytes = str(entry.get("sizeBytes", "")).strip()
|
||
last_modified = str(entry.get("lastModified", "")).strip()
|
||
parts = [normalized_source, str(bucket or "").strip(), relative_path]
|
||
if version_id:
|
||
parts.append(f"version:{version_id}")
|
||
elif sha256:
|
||
parts.append(f"sha256:{sha256}")
|
||
else:
|
||
parts.extend([f"etag:{etag}", f"size:{size_bytes}", f"lastModified:{last_modified}"])
|
||
raw = "\n".join(parts).encode("utf-8")
|
||
return hashlib.sha256(raw).hexdigest()
|
||
|
||
|
||
def incremental_unit(params: dict[str, str]) -> str:
|
||
return "DIRECTORY" if (params.get("INCREMENTAL_UNIT") or "").strip().upper() == "DIRECTORY" else "OBJECT"
|
||
|
||
|
||
def incremental_directory_depth(params: dict[str, str]) -> int:
|
||
raw = (params.get("INCREMENTAL_DIRECTORY_DEPTH") or "").strip()
|
||
try:
|
||
value = int(raw)
|
||
except Exception:
|
||
value = 1
|
||
return value if value > 0 else 1
|
||
|
||
|
||
def directory_incremental_key(relative_path: str, depth: int) -> str:
|
||
clean_relative = str(relative_path or "").strip().lstrip("/")
|
||
if not clean_relative:
|
||
return "__root__"
|
||
parts = [segment for segment in pathlib.PurePosixPath(clean_relative).parts if segment not in ("", ".")]
|
||
if len(parts) <= 1:
|
||
return f"FILE::{clean_relative}"
|
||
directory_parts = parts[:-1]
|
||
take = min(max(depth, 1), len(directory_parts))
|
||
return "/".join(directory_parts[:take])
|
||
|
||
|
||
def incremental_source_state(params: dict[str, str]) -> dict[str, Any]:
|
||
raw = params.get("SOURCE_STATE_JSON", "").strip()
|
||
if not raw:
|
||
return {"seenFingerprints": []}
|
||
try:
|
||
parsed = json.loads(raw)
|
||
except Exception:
|
||
return {"seenFingerprints": []}
|
||
if not isinstance(parsed, dict):
|
||
return {"seenFingerprints": []}
|
||
seen = parsed.get("seenFingerprints")
|
||
if not isinstance(seen, list):
|
||
parsed["seenFingerprints"] = []
|
||
else:
|
||
parsed["seenFingerprints"] = [str(item).strip() for item in seen if str(item).strip()]
|
||
parsed["incrementalUnit"] = "DIRECTORY" if str(parsed.get("incrementalUnit", "")).strip().upper() == "DIRECTORY" else "OBJECT"
|
||
try:
|
||
parsed["directoryDepth"] = int(parsed.get("directoryDepth", 1) or 1)
|
||
except Exception:
|
||
parsed["directoryDepth"] = 1
|
||
if parsed["directoryDepth"] <= 0:
|
||
parsed["directoryDepth"] = 1
|
||
return parsed
|
||
|
||
|
||
def apply_incremental_source_filter(
|
||
entries: list[dict[str, Any]],
|
||
data_source: str,
|
||
bucket: str,
|
||
params: dict[str, str],
|
||
source_path: str,
|
||
prefix: str,
|
||
pattern: str,
|
||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||
state = incremental_source_state(params)
|
||
seen = {str(item).strip() for item in state.get("seenFingerprints", []) if str(item).strip()}
|
||
scan_started_at = now_iso()
|
||
mode = incremental_unit(params)
|
||
depth = incremental_directory_depth(params)
|
||
emitted: list[dict[str, Any]] = []
|
||
updated_seen = set(seen)
|
||
if mode == "DIRECTORY":
|
||
grouped_entries: dict[str, list[dict[str, Any]]] = {}
|
||
for entry in entries:
|
||
group_key = directory_incremental_key(str(entry.get("path", "")).strip(), depth)
|
||
grouped_entries.setdefault(group_key, []).append(entry)
|
||
for group_key in sorted(grouped_entries.keys()):
|
||
group_entries = grouped_entries[group_key]
|
||
if group_key in updated_seen:
|
||
continue
|
||
updated_seen.add(group_key)
|
||
for entry in group_entries:
|
||
enriched = dict(entry)
|
||
enriched["fingerprint"] = source_entry_fingerprint(data_source, bucket, entry)
|
||
enriched["incrementalKey"] = group_key
|
||
emitted.append(enriched)
|
||
else:
|
||
for entry in entries:
|
||
fingerprint = source_entry_fingerprint(data_source, bucket, entry)
|
||
enriched = dict(entry)
|
||
enriched["fingerprint"] = fingerprint
|
||
enriched["incrementalKey"] = fingerprint
|
||
if fingerprint in updated_seen:
|
||
continue
|
||
updated_seen.add(fingerprint)
|
||
emitted.append(enriched)
|
||
scan_completed_at = now_iso()
|
||
checkpoint = {
|
||
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
|
||
"scanMode": "INCREMENTAL",
|
||
"incrementalUnit": mode,
|
||
"directoryDepth": depth,
|
||
"dataSource": data_source,
|
||
"sourcePath": source_path,
|
||
"bucket": bucket,
|
||
"prefix": prefix,
|
||
"pattern": pattern,
|
||
"seenFingerprints": sorted(updated_seen),
|
||
"listedCount": len(entries),
|
||
"newCount": len(emitted),
|
||
"scanStartedAt": scan_started_at,
|
||
"scanCompletedAt": scan_completed_at,
|
||
}
|
||
return emitted, checkpoint
|
||
|
||
|
||
def preview_payload_for_remote_entry(relative_path: str, mime_type: str, preview_url: str) -> dict[str, Any] | None:
|
||
normalized_preview_url = text_value(preview_url).strip()
|
||
if not normalized_preview_url:
|
||
return None
|
||
suffix = pathlib.PurePosixPath(relative_path).suffix.lower()
|
||
kind = "binary"
|
||
if suffix in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
|
||
kind = "video"
|
||
elif suffix in IMAGE_FILE_SUFFIXES or mime_type.startswith("image/"):
|
||
kind = "image"
|
||
elif suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
|
||
kind = "audio"
|
||
return {"preview": {"kind": kind, "format": suffix.lstrip(".") or kind, "url": normalized_preview_url}}
|
||
|
||
|
||
def source_entry_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
entry: dict[str, Any],
|
||
source_path: str,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
relative_path = str(entry.get("path", "")).strip().lstrip("/")
|
||
source = manifest_source_ref(entry, relative_path, source_path)
|
||
storage_path = str(source.get("path", "")).strip() or source_path_join(source_path, relative_path)
|
||
mime_type = str(entry.get("mimeType", "")).strip() or mimetypes.guess_type(relative_path)[0] or ""
|
||
artifact_type = infer_port_type(pathlib.Path(relative_path))
|
||
payload: dict[str, Any] | None = None
|
||
if "data" in entry:
|
||
payload = normalize_json_payload(entry.get("data"))
|
||
else:
|
||
preview_payload = preview_payload_for_remote_entry(relative_path, mime_type, str(source.get("previewURL", "")).strip())
|
||
payload = preview_payload if preview_payload else None
|
||
if payload is not None:
|
||
payload["relativePath"] = relative_path
|
||
if source:
|
||
payload["source"] = source
|
||
metadata = clean_metadata({"mimeType": mime_type, "sha256": entry.get("sha256", "")})
|
||
return build_virtual_artifact(
|
||
run_id,
|
||
node_id,
|
||
pathlib.PurePosixPath(relative_path).name or "artifact",
|
||
storage_path,
|
||
artifact_type,
|
||
int(entry.get("sizeBytes", 0) or 0),
|
||
data=payload,
|
||
metadata=metadata,
|
||
output=output,
|
||
)
|
||
|
||
|
||
def collect_output_artifacts(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any] | None = None,
|
||
output: dict[str, Any] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
artifacts: list[dict[str, Any]] = []
|
||
for path in sorted(output_dir.rglob("*")):
|
||
if path.is_dir():
|
||
continue
|
||
artifact_type = infer_port_type(path)
|
||
mime_type, _ = mimetypes.guess_type(path.name)
|
||
if artifact_type == "json":
|
||
payload = load_payload_from_file(path)
|
||
else:
|
||
payload = extract_preview_data(path, artifact_type, mime_type or "")
|
||
binding = output
|
||
if binding is None and node_data is not None:
|
||
binding = output_binding(node_data)
|
||
artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding))
|
||
return artifacts
|
||
|
||
|
||
def hyperparams(node_data: dict[str, Any]) -> dict[str, str]:
|
||
values: dict[str, str] = {}
|
||
for item in node_data.get("hyperparams", []) or []:
|
||
key = str(item.get("key", "")).strip()
|
||
if not key:
|
||
continue
|
||
values[key] = str(item.get("value", ""))
|
||
return values
|
||
|
||
|
||
def parse_json_param(params: dict[str, str], key: str, default: Any) -> Any:
|
||
raw = params.get(key, "").strip()
|
||
if not raw:
|
||
return default
|
||
try:
|
||
return json.loads(raw)
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
return [load_artifact_payload(item) for item in inputs]
|
||
|
||
|
||
def collect_items(payloads: list[dict[str, Any]]) -> list[Any]:
|
||
items: list[Any] = []
|
||
for payload in payloads:
|
||
if isinstance(payload.get("items"), list):
|
||
items.extend(payload["items"])
|
||
elif isinstance(payload.get("entries"), list):
|
||
items.extend(payload["entries"])
|
||
elif isinstance(payload.get("rows"), list):
|
||
items.extend(payload["rows"])
|
||
else:
|
||
items.append(payload)
|
||
return items
|
||
|
||
|
||
def normalize_mongo_document(item: Any) -> dict[str, Any]:
|
||
if isinstance(item, dict):
|
||
if set(item.keys()) == {"value"} and isinstance(item.get("value"), dict):
|
||
return json_compatible(item["value"])
|
||
return json_compatible(item)
|
||
return {"value": json_compatible(item)}
|
||
|
||
|
||
def documents_from_inputs(payloads: list[dict[str, Any]], document_path: str) -> list[dict[str, Any]]:
|
||
raw_items: list[Any] = []
|
||
path = str(document_path or "").strip()
|
||
if path:
|
||
for payload in payloads:
|
||
value = get_field_value(payload, path)
|
||
if isinstance(value, list):
|
||
raw_items.extend(value)
|
||
elif value is not None:
|
||
raw_items.append(value)
|
||
else:
|
||
raw_items = collect_items(payloads)
|
||
return [normalize_mongo_document(item) for item in raw_items]
|
||
|
||
|
||
def build_mongo_client(uri: str):
|
||
try:
|
||
from pymongo import MongoClient
|
||
except Exception as exc:
|
||
raise RuntimeError("pymongo is required for MongoDB 写入算子") from exc
|
||
return MongoClient(uri, serverSelectionTimeoutMS=5000)
|
||
|
||
|
||
def get_field_value(value: Any, field_path: str) -> Any:
|
||
current = value
|
||
for token in [part for part in field_path.split(".") if part]:
|
||
if isinstance(current, dict):
|
||
current = current.get(token)
|
||
continue
|
||
if isinstance(current, list) and token.isdigit():
|
||
index = int(token)
|
||
if 0 <= index < len(current):
|
||
current = current[index]
|
||
continue
|
||
return None
|
||
return current
|
||
|
||
|
||
def compare_values(actual: Any, operator: str, expected: Any) -> bool:
|
||
if operator == "exists":
|
||
return actual is not None
|
||
if operator == "missing":
|
||
return actual is None
|
||
if operator == "contains":
|
||
if isinstance(actual, list):
|
||
return expected in actual
|
||
return str(expected) in str(actual)
|
||
if operator == "in":
|
||
if isinstance(expected, list):
|
||
return actual in expected
|
||
return str(actual) in str(expected)
|
||
if operator in {"gt", "gte", "lt", "lte"}:
|
||
try:
|
||
actual_number = float(actual)
|
||
expected_number = float(expected)
|
||
except Exception:
|
||
return False
|
||
if operator == "gt":
|
||
return actual_number > expected_number
|
||
if operator == "gte":
|
||
return actual_number >= expected_number
|
||
if operator == "lt":
|
||
return actual_number < expected_number
|
||
return actual_number <= expected_number
|
||
if operator == "ne":
|
||
return actual != expected
|
||
return actual == expected
|
||
|
||
|
||
def flatten_context(value: Any, prefix: str = "") -> dict[str, str]:
|
||
context: dict[str, str] = {}
|
||
if isinstance(value, dict):
|
||
for key, item in value.items():
|
||
next_prefix = f"{prefix}.{key}" if prefix else str(key)
|
||
context.update(flatten_context(item, next_prefix))
|
||
if prefix:
|
||
context[prefix] = json.dumps(value, ensure_ascii=False)
|
||
return context
|
||
if isinstance(value, list):
|
||
for index, item in enumerate(value[:8]):
|
||
next_prefix = f"{prefix}.{index}" if prefix else str(index)
|
||
context.update(flatten_context(item, next_prefix))
|
||
if prefix:
|
||
context[prefix] = json.dumps(value, ensure_ascii=False)
|
||
return context
|
||
if prefix:
|
||
context[prefix] = "" if value is None else str(value)
|
||
return context
|
||
|
||
|
||
def render_template_text(template_text: str, context: dict[str, str]) -> str:
|
||
rendered = template_text
|
||
for key, value in context.items():
|
||
rendered = rendered.replace("{{" + key + "}}", value)
|
||
safe_context = {key: value for key, value in context.items()}
|
||
|
||
class SafeDict(dict):
|
||
def __missing__(self, key: str) -> str:
|
||
return "{" + key + "}"
|
||
|
||
try:
|
||
rendered = rendered.format_map(SafeDict(safe_context))
|
||
except Exception:
|
||
pass
|
||
return rendered
|
||
|
||
|
||
def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
for item in collect_items(payloads):
|
||
if isinstance(item, dict):
|
||
rows.append(item)
|
||
else:
|
||
rows.append({"value": item})
|
||
return rows
|
||
|
||
|
||
def write_text_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
filename: str,
|
||
content: str,
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
path = output_dir / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(content, encoding="utf-8")
|
||
mime_type, _ = mimetypes.guess_type(path.name)
|
||
preview = extract_preview_data(path, infer_port_type(path), mime_type or "")
|
||
return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output)
|
||
|
||
|
||
def write_csv_artifact(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
filename: str,
|
||
rows: list[dict[str, Any]],
|
||
output: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
path = output_dir / filename
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
columns: list[str] = []
|
||
for row in rows:
|
||
for key in row.keys():
|
||
if key not in columns:
|
||
columns.append(key)
|
||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(handle, fieldnames=columns or ["value"])
|
||
writer.writeheader()
|
||
if rows:
|
||
writer.writerows(rows)
|
||
preview = preview_tabular_payload(path, ",", "csv")
|
||
return build_artifact(run_id, node_id, path, "file", data=preview, output=output)
|
||
|
||
|
||
def write_video_segment(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
|
||
cv2, _ = require_cv2()
|
||
capture = cv2.VideoCapture(str(source_path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频文件: {source_path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
start_frame = max(0, int(start_second * fps))
|
||
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
|
||
if frame_count > 0:
|
||
end_frame = min(end_frame, frame_count)
|
||
capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
|
||
if not writer.isOpened():
|
||
raise RuntimeError(f"无法创建视频切片: {destination}")
|
||
written = 0
|
||
try:
|
||
for _ in range(start_frame, end_frame):
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
break
|
||
writer.write(frame)
|
||
written += 1
|
||
finally:
|
||
writer.release()
|
||
duration = written / fps if fps > 0 else 0.0
|
||
return {
|
||
"frameRate": round(fps, 3),
|
||
"frameCount": written,
|
||
"resolution": f"{width}x{height}",
|
||
"duration": round(duration, 3),
|
||
}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def ffmpeg_available() -> bool:
|
||
return shutil.which("ffmpeg") is not None
|
||
|
||
|
||
def run_ffmpeg(args: list[str]) -> None:
|
||
completed = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||
if completed.returncode == 0:
|
||
return
|
||
stderr = (completed.stderr or "").strip()
|
||
stdout = (completed.stdout or "").strip()
|
||
message = stderr or stdout or "unknown ffmpeg error"
|
||
raise RuntimeError(f"ffmpeg execution failed: {message}")
|
||
|
||
|
||
def write_video_segment_with_ffmpeg(source_path: pathlib.Path | str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
|
||
if not ffmpeg_available():
|
||
raise RuntimeError("ffmpeg is not available in operator-runner image")
|
||
duration = max(float(end_second) - float(start_second), 0.001)
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
copy_args = [
|
||
"ffmpeg",
|
||
"-nostdin",
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-y",
|
||
"-ss",
|
||
f"{max(float(start_second), 0.0):.3f}",
|
||
"-i",
|
||
str(source_path),
|
||
"-t",
|
||
f"{duration:.3f}",
|
||
"-map",
|
||
"0:v:0",
|
||
"-map",
|
||
"0:a?",
|
||
"-c",
|
||
"copy",
|
||
str(destination),
|
||
]
|
||
reencode_args = [
|
||
"ffmpeg",
|
||
"-nostdin",
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-y",
|
||
"-ss",
|
||
f"{max(float(start_second), 0.0):.3f}",
|
||
"-i",
|
||
str(source_path),
|
||
"-t",
|
||
f"{duration:.3f}",
|
||
"-map",
|
||
"0:v:0",
|
||
"-map",
|
||
"0:a?",
|
||
"-c:v",
|
||
"libx264",
|
||
"-preset",
|
||
"veryfast",
|
||
"-crf",
|
||
"23",
|
||
"-pix_fmt",
|
||
"yuv420p",
|
||
"-c:a",
|
||
"aac",
|
||
str(destination),
|
||
]
|
||
try:
|
||
run_ffmpeg(copy_args)
|
||
except RuntimeError:
|
||
run_ffmpeg(reencode_args)
|
||
metrics = inspect_video_file(destination)
|
||
return {
|
||
"frameRate": metrics.get("frameRate"),
|
||
"frameCount": metrics.get("frameCount"),
|
||
"resolution": metrics.get("resolution"),
|
||
"duration": metrics.get("duration", round(duration, 3)),
|
||
}
|
||
|
||
|
||
def write_video_segments(
|
||
source_path: pathlib.Path | str,
|
||
segments: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
cv2, _ = require_cv2()
|
||
capture = cv2.VideoCapture(str(source_path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频文件: {source_path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
|
||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) or 64
|
||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) or 64
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
normalized_segments: list[dict[str, Any]] = []
|
||
for raw in segments:
|
||
destination = pathlib.Path(raw.get("destination", ""))
|
||
start_second = float(raw.get("startSecond") or 0.0)
|
||
end_second = float(raw.get("endSecond") or 0.0)
|
||
start_frame = max(0, int(start_second * fps))
|
||
end_frame = max(start_frame + 1, int(math.ceil(end_second * fps)))
|
||
if frame_count > 0:
|
||
end_frame = min(end_frame, frame_count)
|
||
normalized_segments.append(
|
||
{
|
||
**raw,
|
||
"destination": destination,
|
||
"startFrame": start_frame,
|
||
"endFrame": end_frame,
|
||
}
|
||
)
|
||
normalized_segments.sort(key=lambda item: (int(item["startFrame"]), str(item["destination"])))
|
||
current_frame = 0
|
||
if normalized_segments:
|
||
initial_frame = int(normalized_segments[0]["startFrame"])
|
||
if initial_frame > 0:
|
||
capture.set(cv2.CAP_PROP_POS_FRAMES, initial_frame)
|
||
current_frame = initial_frame
|
||
results: list[dict[str, Any]] = []
|
||
for segment in normalized_segments:
|
||
start_frame = int(segment["startFrame"])
|
||
end_frame = int(segment["endFrame"])
|
||
destination = pathlib.Path(segment["destination"])
|
||
if current_frame < start_frame:
|
||
while current_frame < start_frame:
|
||
ok, _ = capture.read()
|
||
if not ok:
|
||
current_frame = end_frame
|
||
break
|
||
current_frame += 1
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
|
||
if not writer.isOpened():
|
||
raise RuntimeError(f"无法创建视频切片: {destination}")
|
||
written = 0
|
||
try:
|
||
while current_frame < end_frame:
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
break
|
||
writer.write(frame)
|
||
written += 1
|
||
current_frame += 1
|
||
finally:
|
||
writer.release()
|
||
results.append(
|
||
{
|
||
**segment,
|
||
"frameRate": round(fps, 3),
|
||
"frameCount": written,
|
||
"resolution": f"{width}x{height}",
|
||
"duration": round(written / fps, 3) if fps > 0 else 0.0,
|
||
}
|
||
)
|
||
return results
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def write_video_segment_from_remote_url(source_url: str, destination: pathlib.Path, start_second: float, end_second: float) -> dict[str, Any]:
|
||
return write_video_segment(source_url, destination, start_second, end_second)
|
||
|
||
|
||
def annotate_video_clip(path: pathlib.Path, frame_interval_seconds: float) -> dict[str, Any]:
|
||
cv2, np = require_cv2()
|
||
capture = cv2.VideoCapture(str(path))
|
||
if not capture.isOpened():
|
||
raise RuntimeError(f"无法打开视频切片: {path}")
|
||
try:
|
||
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) or 10.0
|
||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
||
duration = (frame_count / fps) if fps > 0 and frame_count > 0 else 0.0
|
||
step = max(1, int(round(max(frame_interval_seconds, 0.1) * fps)))
|
||
previous_gray = None
|
||
samples: list[tuple[float, float]] = []
|
||
objects: set[str] = set()
|
||
frame_index = 0
|
||
while True:
|
||
ok, frame = capture.read()
|
||
if not ok:
|
||
break
|
||
if frame_index % step != 0:
|
||
frame_index += 1
|
||
continue
|
||
timestamp = frame_index / fps if fps > 0 else 0.0
|
||
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||
motion = 0.0
|
||
if previous_gray is not None:
|
||
diff = cv2.absdiff(previous_gray, gray)
|
||
motion = float(diff.mean() / 255.0)
|
||
previous_gray = gray
|
||
ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
|
||
mask = cv2.inRange(ycrcb, np.array((0, 133, 77), dtype=np.uint8), np.array((255, 173, 127), dtype=np.uint8))
|
||
if float(mask.mean() / 255.0) > 0.1:
|
||
objects.add("手部")
|
||
_, threshold = cv2.threshold(gray, 24, 255, cv2.THRESH_BINARY)
|
||
contours, _ = cv2.findContours(threshold, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||
if any(cv2.contourArea(contour) >= 64 for contour in contours):
|
||
objects.add("前景目标")
|
||
samples.append((timestamp, motion))
|
||
frame_index += 1
|
||
if not samples:
|
||
samples.append((0.0, 0.0))
|
||
|
||
def label_for_motion(motion: float) -> tuple[str, float]:
|
||
if motion < 0.015:
|
||
return "静止观察", 0.88
|
||
if motion < 0.045:
|
||
return "平稳活动", 0.79
|
||
return "快速移动", 0.71
|
||
|
||
actions: list[dict[str, Any]] = []
|
||
last_timestamp = duration
|
||
for timestamp, motion in samples:
|
||
label, confidence = label_for_motion(motion)
|
||
last_timestamp = timestamp
|
||
if not actions:
|
||
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
|
||
continue
|
||
current = actions[-1]
|
||
if current["label"] != label:
|
||
current["end"] = round(timestamp, 3)
|
||
actions.append({"label": label, "confidence": confidence, "start": round(timestamp, 3), "end": round(timestamp, 3)})
|
||
else:
|
||
current["end"] = round(timestamp, 3)
|
||
current["confidence"] = round(min(float(current["confidence"]), confidence), 3)
|
||
if actions:
|
||
actions[-1]["end"] = round(max(duration, last_timestamp), 3)
|
||
min_confidence = min(float(item["confidence"]) for item in actions) if actions else 0.0
|
||
return {
|
||
"duration": round(duration, 3),
|
||
"actions": actions,
|
||
"objects": sorted(objects),
|
||
"minConfidence": round(min_confidence, 3),
|
||
"frameIntervalSeconds": frame_interval_seconds,
|
||
}
|
||
finally:
|
||
capture.release()
|
||
|
||
|
||
def builtin_source(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT"))
|
||
data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper()
|
||
if not data_source:
|
||
data_source = "INLINE" if inline_source else "BOS"
|
||
source_path = params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "").strip()
|
||
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
|
||
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
|
||
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
|
||
bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
scan_mode = (params.get("SCAN_MODE") or "").strip().upper()
|
||
unit = incremental_unit(params)
|
||
depth = incremental_directory_depth(params)
|
||
source_bucket, source_prefix = parse_bos_path(source_path)
|
||
if not bucket and source_bucket:
|
||
bucket = source_bucket
|
||
if not prefix and source_prefix:
|
||
prefix = source_prefix
|
||
|
||
entries: list[dict[str, Any]] = []
|
||
resolved_source_path = source_path
|
||
if params.get("SOURCE_JSON"):
|
||
parsed = json.loads(params["SOURCE_JSON"])
|
||
if isinstance(parsed, list):
|
||
entries = [{"value": item} for item in parsed]
|
||
else:
|
||
entries = [{"value": parsed}]
|
||
elif params.get("SOURCE_TEXT"):
|
||
entries = [{"text": params["SOURCE_TEXT"]}]
|
||
elif data_source == "BOS" or (bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key):
|
||
cleaned_prefix = bos_prefix(prefix)
|
||
entries = list_bos_entries(
|
||
bucket,
|
||
cleaned_prefix,
|
||
pattern,
|
||
bos_endpoint,
|
||
bos_access_key_id,
|
||
bos_secret_access_key,
|
||
)
|
||
resolved_source_path = canonical_bos_path(bucket, cleaned_prefix)
|
||
elif source_path:
|
||
entries = list_source_entries(pathlib.Path(source_path), pattern)
|
||
elif prefix and pathlib.Path(prefix).exists():
|
||
entries = list_source_entries(pathlib.Path(prefix), pattern)
|
||
elif inputs:
|
||
entries = [load_artifact_payload(item) for item in inputs]
|
||
|
||
listed_entries = list(entries)
|
||
checkpoint_payload = None
|
||
if scan_mode == "INCREMENTAL":
|
||
entries, checkpoint_payload = apply_incremental_source_filter(
|
||
listed_entries,
|
||
data_source,
|
||
bucket,
|
||
params,
|
||
resolved_source_path,
|
||
prefix,
|
||
pattern,
|
||
)
|
||
|
||
payload = {
|
||
"dataSource": data_source,
|
||
"bucket": bucket,
|
||
"prefix": prefix,
|
||
"endpoint": bos_endpoint,
|
||
"pattern": pattern,
|
||
"scanMode": scan_mode or "FULL",
|
||
"incrementalUnit": unit,
|
||
"directoryDepth": depth,
|
||
"sourcePath": resolved_source_path,
|
||
"sourceFingerprint": str(params.get("SOURCE_FINGERPRINT", "")).strip(),
|
||
"listedCount": len(listed_entries),
|
||
"newCount": len(entries) if checkpoint_payload is not None else len(entries),
|
||
"entries": entries,
|
||
"catalogEntries": listed_entries,
|
||
}
|
||
artifacts = [
|
||
write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))
|
||
]
|
||
if checkpoint_payload is not None:
|
||
artifacts.append(write_json_artifact(run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload))
|
||
|
||
if data_source == "BOS":
|
||
for entry in entries:
|
||
artifacts.append(source_entry_artifact(run_id, node_id, entry, resolved_source_path, output=output_binding(node_data)))
|
||
elif source_path and pathlib.Path(source_path).exists():
|
||
source_root = pathlib.Path(source_path)
|
||
staged_root = output_dir / "_source_tree"
|
||
for entry in entries:
|
||
relative_path = str(entry.get("path", "")).strip()
|
||
if not relative_path:
|
||
continue
|
||
candidate = source_root / relative_path if source_root.is_dir() else source_root
|
||
if not candidate.exists() or not candidate.is_file():
|
||
continue
|
||
destination = staged_root / relative_path
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(candidate, destination)
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, staged_root, node_data))
|
||
|
||
return artifacts
|
||
|
||
|
||
def builtin_merge(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = payloads_from_inputs(inputs)
|
||
merged = merge_dicts(payloads)
|
||
merged["inputCount"] = len(payloads)
|
||
return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_intersection(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = payloads_from_inputs(inputs)
|
||
if not payloads:
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"intersection.json",
|
||
{"keys": [], "inputCount": 0},
|
||
output=output_binding(node_data),
|
||
)
|
||
]
|
||
key_sets = [set(payload.keys()) for payload in payloads]
|
||
common_keys = sorted(set.intersection(*key_sets)) if key_sets else []
|
||
payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_distribution(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = payloads_from_inputs(inputs)
|
||
items = collect_items(payloads)
|
||
outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}]
|
||
partitions: list[list[Any]] = [[] for _ in outputs]
|
||
for index, item in enumerate(items):
|
||
partitions[index % len(outputs)].append(item)
|
||
artifacts: list[dict[str, Any]] = []
|
||
for index, output in enumerate(outputs):
|
||
filename = f"{output.get('name', f'queue-{index}')}.json"
|
||
payload = {"items": partitions[index], "count": len(partitions[index])}
|
||
artifacts.append(
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
filename,
|
||
payload,
|
||
str(output.get("type", "json")),
|
||
output=output,
|
||
)
|
||
)
|
||
return artifacts
|
||
|
||
|
||
def builtin_upload(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip()
|
||
bucket = str(node_data.get("bucket", "")).strip()
|
||
prefix = str(node_data.get("prefix", "")).strip()
|
||
parsed_bucket, parsed_prefix = parse_bos_path(upload_path)
|
||
if parsed_bucket:
|
||
bucket = parsed_bucket
|
||
if parsed_prefix or upload_path.endswith("/"):
|
||
prefix = parsed_prefix
|
||
data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper()
|
||
bos_endpoint = params.get("BOS_ENDPOINT", "").strip()
|
||
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip()
|
||
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip()
|
||
copied_paths: list[str] = []
|
||
bundle_inputs: list[dict[str, Any]] = []
|
||
staging_dir = output_dir / "uploaded"
|
||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||
package_manifest_payload = first_payload_with_keys(inputs, "packageCount", "items")
|
||
upload_inputs = list(inputs)
|
||
if package_manifest_payload is not None:
|
||
package_names = package_related_artifact_names(package_manifest_payload)
|
||
upload_inputs = [
|
||
artifact
|
||
for artifact in inputs
|
||
if str(artifact.get("name", "")).strip() not in package_names
|
||
]
|
||
upload_inputs.extend(
|
||
materialize_package_artifacts_from_manifest(
|
||
run_id,
|
||
node_id,
|
||
staging_dir,
|
||
package_manifest_payload,
|
||
node_data,
|
||
)
|
||
)
|
||
|
||
if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key:
|
||
client, scheme = build_bos_client(bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
||
temp_paths = [
|
||
str(item.get("path", "")).strip()
|
||
for item in upload_inputs
|
||
if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip()
|
||
]
|
||
input_root = common_input_root(temp_paths)
|
||
uploaded_objects: list[dict[str, Any]] = []
|
||
remote_artifacts: list[dict[str, Any]] = []
|
||
|
||
for index, artifact in enumerate(upload_inputs):
|
||
payload = load_artifact_payload(artifact)
|
||
bundle_inputs.append(payload)
|
||
fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}"
|
||
relative_path = upload_relative_path(artifact, input_root, fallback_name)
|
||
temp_source = materialized_upload_source(artifact, staging_dir, relative_path)
|
||
if temp_source is None or not temp_source.exists():
|
||
continue
|
||
|
||
mime_type = (
|
||
str((artifact.get("metadata") or {}).get("mimeType", "")).strip()
|
||
if isinstance(artifact.get("metadata"), dict)
|
||
else ""
|
||
) or mimetypes.guess_type(temp_source.name)[0] or ""
|
||
object_key = bos_key_join(prefix, relative_path)
|
||
put_kwargs = {"content_type": mime_type} if mime_type else {}
|
||
try:
|
||
size_bytes = temp_source.stat().st_size
|
||
except OSError:
|
||
size_bytes = 0
|
||
log(
|
||
f"[operator-runner][info] upload starts: {temp_source} -> {canonical_bos_path(bucket, object_key)} "
|
||
f"({size_bytes} bytes)"
|
||
)
|
||
client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs)
|
||
remote_path = canonical_bos_path(bucket, object_key)
|
||
log(
|
||
f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} "
|
||
f"({size_bytes} bytes)"
|
||
)
|
||
preview_url = ""
|
||
if pathlib.PurePosixPath(relative_path).suffix.lower() in VIDEO_FILE_SUFFIXES or mime_type.startswith("video/"):
|
||
preview_url = text_value(
|
||
client.generate_pre_signed_url(
|
||
bucket,
|
||
object_key,
|
||
expiration_in_seconds=3600,
|
||
protocol=scheme,
|
||
)
|
||
).strip()
|
||
|
||
uploaded_objects.append(
|
||
clean_metadata(
|
||
{
|
||
"name": pathlib.PurePosixPath(relative_path).name,
|
||
"relativePath": relative_path,
|
||
"path": remote_path,
|
||
"previewURL": preview_url,
|
||
"sizeBytes": temp_source.stat().st_size,
|
||
"mimeType": mime_type,
|
||
}
|
||
)
|
||
or {}
|
||
)
|
||
|
||
artifact_type = str(artifact.get("type", "")).strip() or infer_port_type(temp_source)
|
||
artifact_payload: dict[str, Any] | None = None
|
||
if artifact_type == "json":
|
||
loaded = load_payload_from_file(temp_source)
|
||
if loaded is not None:
|
||
artifact_payload = normalize_json_payload(loaded)
|
||
else:
|
||
artifact_payload = preview_payload_for_remote_entry(relative_path, mime_type, preview_url)
|
||
if artifact_payload is None:
|
||
artifact_payload = extract_preview_data(temp_source, artifact_type, mime_type)
|
||
if artifact_payload is not None:
|
||
artifact_payload["relativePath"] = relative_path
|
||
artifact_payload["source"] = clean_metadata(
|
||
{
|
||
"path": remote_path,
|
||
"previewURL": preview_url,
|
||
"relativePath": relative_path,
|
||
"mimeType": mime_type,
|
||
}
|
||
) or {}
|
||
|
||
source_size_bytes = temp_source.stat().st_size
|
||
source_sha256, source_sha256_metadata = artifact_sha256(temp_source, source_size_bytes)
|
||
|
||
remote_artifacts.append(
|
||
build_virtual_artifact(
|
||
run_id,
|
||
node_id,
|
||
pathlib.PurePosixPath(relative_path).name,
|
||
remote_path,
|
||
artifact_type,
|
||
source_size_bytes,
|
||
data=artifact_payload,
|
||
metadata={
|
||
**source_sha256_metadata,
|
||
"sha256": source_sha256,
|
||
"mimeType": mime_type,
|
||
},
|
||
output=output_binding(node_data),
|
||
)
|
||
)
|
||
|
||
payload = {
|
||
"copiedPaths": copied_paths,
|
||
"uploadedCount": len(uploaded_objects),
|
||
"uploadedObjects": uploaded_objects,
|
||
"items": bundle_inputs,
|
||
"inputCount": len(inputs),
|
||
"dataSource": "BOS",
|
||
"bucket": bucket,
|
||
"prefix": prefix,
|
||
"destinationPath": canonical_bos_path(bucket, prefix),
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
|
||
artifacts.extend(remote_artifacts)
|
||
return artifacts
|
||
|
||
for artifact in upload_inputs:
|
||
payload = load_artifact_payload(artifact)
|
||
bundle_inputs.append(payload)
|
||
source_path = artifact.get("path")
|
||
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
|
||
source = pathlib.Path(source_path)
|
||
destination = staging_dir / source.name
|
||
if source.resolve() != destination.resolve():
|
||
shutil.copy2(source, destination)
|
||
copied_paths.append(str(destination))
|
||
|
||
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir))
|
||
return artifacts
|
||
|
||
|
||
def builtin_json_extract(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_path = params.get("FIELD_PATH", "").strip()
|
||
payloads = payloads_from_inputs(inputs)
|
||
payload = payloads[0] if payloads else {}
|
||
extracted = get_field_value(payload, field_path) if field_path else payload
|
||
result = {"fieldPath": field_path, "value": extracted}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_json_filter(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_path = params.get("FIELD_PATH", "").strip()
|
||
operator = params.get("OPERATOR", "eq").strip().lower() or "eq"
|
||
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
|
||
rows = collect_items(payloads_from_inputs(inputs))
|
||
filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
|
||
payload = {
|
||
"fieldPath": field_path,
|
||
"operator": operator,
|
||
"expected": expected,
|
||
"items": filtered,
|
||
"count": len(filtered),
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_json_map(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_map = parse_json_param(params, "FIELD_MAP_JSON", {})
|
||
rows = rows_from_payloads(payloads_from_inputs(inputs))
|
||
mapped = [{str(field_map.get(key, key)): value for key, value in row.items()} for row in rows]
|
||
payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_mongodb_write(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
mongo_target = params.get("MONGO_TARGET", "").strip().lower() or "business"
|
||
explicit_mongo_uri = params.get("MONGO_URI", "").strip() or os.getenv("MONGO_URI", "").strip()
|
||
explicit_mongo_db = params.get("MONGO_DB", "").strip() or os.getenv("MONGO_DB", "").strip()
|
||
collection_name = params.get("MONGO_COLLECTION", "").strip()
|
||
mode = params.get("MONGO_MODE", "insert_many").strip().lower() or "insert_many"
|
||
upsert_key = params.get("UPSERT_KEY", "_id").strip() or "_id"
|
||
document_path = params.get("DOCUMENT_PATH", "").strip()
|
||
|
||
mongo_uri = explicit_mongo_uri
|
||
mongo_db = explicit_mongo_db
|
||
if not mongo_uri:
|
||
if mongo_target == "platform":
|
||
mongo_uri = os.getenv("PLATFORM_MONGO_URI", "").strip()
|
||
mongo_db = mongo_db or os.getenv("PLATFORM_MONGO_DB", "").strip()
|
||
else:
|
||
mongo_uri = os.getenv("BUSINESS_MONGO_URI", "").strip()
|
||
mongo_db = mongo_db or os.getenv("BUSINESS_MONGO_DB", "").strip()
|
||
|
||
if not mongo_uri:
|
||
raise RuntimeError(f"MongoDB 写入缺少可用连接:target={mongo_target}")
|
||
if not mongo_db:
|
||
raise RuntimeError(f"MongoDB 写入缺少数据库名:target={mongo_target}")
|
||
if not collection_name:
|
||
raise RuntimeError("MONGO_COLLECTION is required for MongoDB 写入")
|
||
if mongo_target == "platform" and collection_name.lower() in {item.lower() for item in RESERVED_MONGO_COLLECTIONS}:
|
||
raise RuntimeError(f"collection {collection_name} is reserved and cannot be written by workflow operators")
|
||
if mode not in {"insert_many", "upsert"}:
|
||
raise RuntimeError("MONGO_MODE must be insert_many or upsert")
|
||
|
||
documents = documents_from_inputs(payloads_from_inputs(inputs), document_path)
|
||
if not documents:
|
||
raise RuntimeError("MongoDB 写入未收到可写入的文档")
|
||
|
||
client = build_mongo_client(mongo_uri)
|
||
try:
|
||
collection = client[mongo_db][collection_name]
|
||
summary: dict[str, Any] = {
|
||
"target": mongo_target,
|
||
"database": mongo_db,
|
||
"collection": collection_name,
|
||
"mode": mode,
|
||
"documentCount": len(documents),
|
||
}
|
||
if document_path:
|
||
summary["documentPath"] = document_path
|
||
if mode == "insert_many":
|
||
result = collection.insert_many(documents, ordered=False)
|
||
summary["insertedCount"] = len(getattr(result, "inserted_ids", []) or [])
|
||
else:
|
||
processed = 0
|
||
matched_count = 0
|
||
modified_count = 0
|
||
upserted_ids: list[Any] = []
|
||
for document in documents:
|
||
key_value = get_field_value(document, upsert_key)
|
||
if key_value is None:
|
||
raise RuntimeError(f"document is missing UPSERT_KEY={upsert_key}")
|
||
result = collection.replace_one({upsert_key: key_value}, document, upsert=True)
|
||
processed += 1
|
||
matched_count += int(getattr(result, "matched_count", 0) or 0)
|
||
modified_count += int(getattr(result, "modified_count", 0) or 0)
|
||
upserted_id = getattr(result, "upserted_id", None)
|
||
if upserted_id is not None:
|
||
upserted_ids.append(json_compatible(upserted_id))
|
||
summary["upsertKey"] = upsert_key
|
||
summary["upsertedCount"] = processed
|
||
summary["matchedCount"] = matched_count
|
||
summary["modifiedCount"] = modified_count
|
||
if upserted_ids:
|
||
summary["upsertedIds"] = upserted_ids
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"mongodb-write-result.json",
|
||
summary,
|
||
output=output_binding(node_data),
|
||
)
|
||
]
|
||
finally:
|
||
close = getattr(client, "close", None)
|
||
if callable(close):
|
||
close()
|
||
|
||
|
||
def builtin_text_template(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
template_text = params.get("TEMPLATE_TEXT", "").strip()
|
||
output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt"
|
||
payloads = payloads_from_inputs(inputs)
|
||
context = flatten_context(payloads[0] if payloads else {})
|
||
rendered = render_template_text(template_text, context)
|
||
binding = output_binding(node_data)
|
||
manifest = write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"rendered-template.json",
|
||
{"template": template_text, "context": context, "outputName": output_name},
|
||
output=binding,
|
||
)
|
||
return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)]
|
||
|
||
|
||
def builtin_csv_to_json(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
payload: dict[str, Any]
|
||
if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES:
|
||
delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()]
|
||
columns, rows = load_tabular_rows(source_path, delimiter)
|
||
payload = {
|
||
"sourcePath": str(source_path),
|
||
"columns": columns,
|
||
"items": rows,
|
||
"rowCount": len(rows),
|
||
}
|
||
else:
|
||
for artifact in inputs:
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if preview.get("kind") == "table":
|
||
payload = {
|
||
"sourcePath": str(artifact.get("path", "")),
|
||
"columns": preview.get("columns", []),
|
||
"items": preview.get("rows", []),
|
||
"rowCount": preview.get("rowCount", len(preview.get("rows", []))),
|
||
"previewTruncated": preview.get("truncated", False),
|
||
}
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"table.json",
|
||
payload,
|
||
output=output_binding(node_data),
|
||
)
|
||
]
|
||
raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件")
|
||
return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_json_to_csv(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
rows = rows_from_payloads(payloads_from_inputs(inputs))
|
||
binding = output_binding(node_data)
|
||
summary = write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
"csv-export-summary.json",
|
||
{"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})},
|
||
output=binding,
|
||
)
|
||
return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)]
|
||
|
||
|
||
def builtin_directory_scan(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
root = source_path if source_path.is_dir() else source_path.parent
|
||
payload = {
|
||
"sourcePath": str(source_path),
|
||
"scanRoot": str(root),
|
||
"summary": summarize_directory_tree(root),
|
||
}
|
||
elif inputs:
|
||
payload = {
|
||
"sourcePath": "",
|
||
"scanRoot": "",
|
||
"summary": summarize_input_artifacts(inputs),
|
||
}
|
||
else:
|
||
raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录")
|
||
return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_xplatform_index(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payload = load_xplatform_index_payload(hyperparams(node_data), inputs, output_dir)
|
||
return [write_json_artifact(run_id, node_id, output_dir, "dataset-index.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_collection_validate(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
require_intrinsics = parse_bool_param(params, "REQUIRE_INTRINSICS", False)
|
||
payload = first_payload_with_keys(inputs, "collections", "items")
|
||
if payload is None:
|
||
payload = load_xplatform_index_payload(params, inputs, output_dir)
|
||
collections = []
|
||
valid_collection_count = 0
|
||
for collection in payload.get("collections", []):
|
||
if not isinstance(collection, dict):
|
||
continue
|
||
missing_required = list(collection.get("missingRequiredFiles", []))
|
||
if require_intrinsics and not collection.get("hasIntrinsics", False):
|
||
missing_required.append("intrinsics.json")
|
||
valid = len(missing_required) == 0 and int(collection.get("videoCount", 0) or 0) > 0
|
||
if valid:
|
||
valid_collection_count += 1
|
||
collections.append({**collection, "missingRequiredFiles": missing_required, "valid": valid})
|
||
items = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
collection_id = str(item.get("collectionId", "")).strip()
|
||
items.append({**item, "collectionValid": any(c.get("id") == collection_id and c.get("valid", False) for c in collections)})
|
||
result = {
|
||
**payload,
|
||
"datasetType": "x-platform-validation",
|
||
"collections": collections,
|
||
"items": items,
|
||
"validCollectionCount": valid_collection_count,
|
||
"invalidCollectionCount": max(len(collections) - valid_collection_count, 0),
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "dataset-validation.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_video_quality(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
min_clarity = parse_float_param(params, "MIN_CLARITY", 0.8)
|
||
max_shake = parse_float_param(params, "MAX_SHAKE", 0.1)
|
||
max_hand_occlusion = parse_float_param(params, "MAX_HAND_OCCLUSION", 0.1)
|
||
clarity_scale = parse_float_param(params, "CLARITY_SCALE", 35.0)
|
||
payload = first_payload_with_keys(inputs, "items")
|
||
if payload is None:
|
||
payload = load_xplatform_index_payload(params, inputs, output_dir)
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
items = []
|
||
passed_count = 0
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
source_path = source_path_from_item(item, str(item.get("filename", "video.mp4")), output_dir, archive_sources)
|
||
metrics = analyze_video_quality_metrics(source_path, clarity_scale)
|
||
passed = (
|
||
float_value(metrics.get("clarity"), 0.0) >= min_clarity
|
||
and float_value(metrics.get("shake"), 1.0) <= max_shake
|
||
and float_value(metrics.get("handOcclusion"), 1.0) <= max_hand_occlusion
|
||
)
|
||
if passed:
|
||
passed_count += 1
|
||
items.append(
|
||
{
|
||
**item,
|
||
"quality": {
|
||
**metrics,
|
||
"passed": passed,
|
||
"thresholds": {
|
||
"minClarity": min_clarity,
|
||
"maxShake": max_shake,
|
||
"maxHandOcclusion": max_hand_occlusion,
|
||
},
|
||
},
|
||
}
|
||
)
|
||
result = {
|
||
**payload,
|
||
"datasetType": "x-platform-quality",
|
||
"items": items,
|
||
"videoCount": len(items),
|
||
"passedCount": passed_count,
|
||
"failedCount": max(len(items) - passed_count, 0),
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "quality-report.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_dataset_filter(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "items")
|
||
if payload is None:
|
||
raise RuntimeError("数据过滤需要上游数据清单输入")
|
||
date_start = normalize_date_value(params.get("DATE_START", ""))
|
||
date_end = normalize_date_value(params.get("DATE_END", ""))
|
||
place = params.get("PLACE", "").strip()
|
||
collection_type = params.get("TYPE", "").strip()
|
||
min_clarity = parse_float_param(params, "MIN_CLARITY", 0.0)
|
||
max_shake = parse_float_param(params, "MAX_SHAKE", 1.0)
|
||
max_hand_occlusion = parse_float_param(params, "MAX_HAND_OCCLUSION", 1.0)
|
||
filtered = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
meta = item.get("meta") if isinstance(item.get("meta"), dict) else {}
|
||
quality = item.get("quality") if isinstance(item.get("quality"), dict) else {}
|
||
if place and str(meta.get("place", "")).strip() != place:
|
||
continue
|
||
if collection_type and str(meta.get("type", "")).strip() != collection_type:
|
||
continue
|
||
if date_start or date_end:
|
||
if not date_in_range(meta.get("date", ""), date_start, date_end):
|
||
continue
|
||
if float_value(quality.get("clarity"), 0.0) < min_clarity:
|
||
continue
|
||
if float_value(quality.get("shake"), 1.0) > max_shake:
|
||
continue
|
||
if float_value(quality.get("handOcclusion"), 1.0) > max_hand_occlusion:
|
||
continue
|
||
if not bool(quality.get("passed", False)):
|
||
continue
|
||
filtered.append(item)
|
||
result = {
|
||
"datasetType": "x-platform-filtered",
|
||
"criteria": {
|
||
"dateStart": date_start,
|
||
"dateEnd": date_end,
|
||
"place": place,
|
||
"type": collection_type,
|
||
"minClarity": min_clarity,
|
||
"maxShake": max_shake,
|
||
"maxHandOcclusion": max_hand_occlusion,
|
||
},
|
||
"count": len(filtered),
|
||
"items": filtered,
|
||
}
|
||
if isinstance(payload.get("archiveSources"), dict):
|
||
result["archiveSources"] = payload["archiveSources"]
|
||
return [write_json_artifact(run_id, node_id, output_dir, "filtered-videos.json", result, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_video_slicing(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "items")
|
||
if payload is None:
|
||
raise RuntimeError("视频切片需要上游过滤结果输入")
|
||
segment_duration = max(1, parse_int_param(params, "SEGMENT_DURATION_SECONDS", 300))
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
clips: list[dict[str, Any]] = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
source_filename = str(item.get("filename", "")).strip() or "video.mp4"
|
||
source_path = source_path_from_item(item, source_filename, output_dir, archive_sources)
|
||
metrics = inspect_video_file(source_path)
|
||
duration = float(item.get("duration") or metrics.get("duration") or 0.0)
|
||
if duration <= 0:
|
||
duration = float(metrics.get("duration") or 0.0)
|
||
if duration <= 0:
|
||
duration = float(segment_duration)
|
||
clip_count = max(1, int(math.ceil(duration / segment_duration)))
|
||
stem = pathlib.Path(source_filename).stem
|
||
clip_specs: list[dict[str, Any]] = []
|
||
for clip_index in range(clip_count):
|
||
clip_start = clip_index * segment_duration
|
||
clip_end = min(duration, clip_start + segment_duration)
|
||
clip_name = f"{stem}_clip{clip_index + 1:03d}.mp4"
|
||
clip_path = output_dir / "clips" / clip_name
|
||
clip_specs.append(
|
||
{
|
||
"clipIndex": clip_index,
|
||
"clipStart": clip_start,
|
||
"clipEnd": clip_end,
|
||
"clipName": clip_name,
|
||
"clipPath": clip_path,
|
||
"destination": clip_path,
|
||
"startSecond": clip_start,
|
||
"endSecond": clip_end,
|
||
}
|
||
)
|
||
clip_metrics_list: list[dict[str, Any]] | None = None
|
||
if ffmpeg_available():
|
||
try:
|
||
log(f"[operator-runner][info] video_slicing uses ffmpeg for {source_filename} ({clip_count} clip(s))")
|
||
clip_metrics_list = []
|
||
for clip_spec in clip_specs:
|
||
clip_metrics_list.append(
|
||
{
|
||
**clip_spec,
|
||
**write_video_segment_with_ffmpeg(
|
||
source_path,
|
||
pathlib.Path(clip_spec["clipPath"]),
|
||
float(clip_spec["clipStart"]),
|
||
float(clip_spec["clipEnd"]),
|
||
),
|
||
}
|
||
)
|
||
except RuntimeError as exc:
|
||
log(f"[operator-runner][warn] ffmpeg video slicing failed for {source_filename}, fallback to opencv: {exc}")
|
||
clip_metrics_list = None
|
||
if clip_metrics_list is None:
|
||
if len(clip_specs) > 1:
|
||
clip_metrics_list = write_video_segments(source_path, clip_specs)
|
||
else:
|
||
only = clip_specs[0]
|
||
clip_metrics_list = [
|
||
{
|
||
**only,
|
||
**write_video_segment(source_path, only["clipPath"], float(only["clipStart"]), float(only["clipEnd"])),
|
||
}
|
||
]
|
||
for clip_spec in clip_metrics_list:
|
||
clip_start = float(clip_spec["clipStart"])
|
||
clip_end = float(clip_spec["clipEnd"])
|
||
clip_name = str(clip_spec["clipName"])
|
||
clip_path = pathlib.Path(clip_spec["clipPath"])
|
||
clips.append(
|
||
{
|
||
"clipId": pathlib.Path(clip_name).stem,
|
||
"clipName": clip_name,
|
||
"clipPath": str(clip_path),
|
||
"sourceVideo": source_filename,
|
||
"sourcePath": str(source_path),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
"collectionId": item.get("collectionId", ""),
|
||
"meta": item.get("meta", {}),
|
||
"quality": item.get("quality", {}),
|
||
"startSecond": round(clip_start, 3),
|
||
"endSecond": round(clip_end, 3),
|
||
"duration": clip_spec.get("duration", round(max(clip_end - clip_start, 0.0), 3)),
|
||
"resolution": clip_spec.get("resolution"),
|
||
"frameRate": clip_spec.get("frameRate"),
|
||
}
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-clips",
|
||
"segmentDurationSeconds": segment_duration,
|
||
"clipCount": len(clips),
|
||
"items": clips,
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "clips-manifest.json", manifest, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "clips", node_data=node_data, output=output_binding(node_data)))
|
||
return artifacts
|
||
|
||
|
||
def builtin_video_annotation(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "clipCount", "items")
|
||
if payload is None:
|
||
raise RuntimeError("视频标注需要上游切片结果输入")
|
||
frame_interval = max(0.1, parse_float_param(params, "FRAME_INTERVAL_SECONDS", 1.0))
|
||
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
items = []
|
||
for clip in payload.get("items", []):
|
||
if not isinstance(clip, dict):
|
||
continue
|
||
clip_name = str(clip.get("clipName", "")).strip() or pathlib.Path(str(clip.get("clipPath", ""))).name
|
||
artifact = clip_artifacts.get(clip_name)
|
||
clip_path: pathlib.Path | None = None
|
||
if artifact is not None:
|
||
try:
|
||
clip_path = source_path_from_item(
|
||
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
|
||
clip_name,
|
||
output_dir,
|
||
archive_sources,
|
||
)
|
||
except RuntimeError:
|
||
clip_path = None
|
||
if clip_path is None:
|
||
direct_clip_path = str(clip.get("clipPath", "")).strip()
|
||
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
|
||
clip_path = pathlib.Path(direct_clip_path)
|
||
if clip_path is None:
|
||
clip_path = materialize_clip_from_source(clip, output_dir, archive_sources)
|
||
if clip_path is None or not clip_path.exists():
|
||
raise RuntimeError(f"无法获取真实输入文件: {clip_name}")
|
||
annotation = annotate_video_clip(clip_path, frame_interval)
|
||
annotation_filename = f"{pathlib.Path(clip_name).stem}_annotation.json"
|
||
annotation_payload = {
|
||
"clipId": clip.get("clipId", pathlib.Path(clip_name).stem),
|
||
"sourceVideo": clip.get("sourceVideo", ""),
|
||
**annotation,
|
||
}
|
||
items.append({**clip, "annotation": annotation_payload, "annotationFile": annotation_filename})
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir / "annotations",
|
||
annotation_filename,
|
||
annotation_payload,
|
||
output=output_binding(node_data),
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-annotations",
|
||
"annotationCount": len(items),
|
||
"items": items,
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "annotations-manifest.json", manifest, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "annotations", node_data=node_data, output=output_binding(node_data)))
|
||
return artifacts
|
||
|
||
|
||
def builtin_result_package(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payload = first_payload_with_keys(inputs, "annotationCount", "items")
|
||
if payload is None:
|
||
raise RuntimeError("成品打包需要上游标注结果输入")
|
||
clip_artifacts = {str(item.get("name", "")).strip(): item for item in inputs if str(item.get("name", "")).strip().lower().endswith(".mp4")}
|
||
archive_sources = payload.get("archiveSources", {}) if isinstance(payload.get("archiveSources"), dict) else {}
|
||
packages = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
clip_name = str(item.get("clipName", "")).strip()
|
||
clip_id = str(item.get("clipId", "")).strip() or pathlib.Path(clip_name).stem
|
||
package_name = f"{clip_id}_package"
|
||
package_dir = output_dir / "packages" / package_name
|
||
package_dir.mkdir(parents=True, exist_ok=True)
|
||
artifact = clip_artifacts.get(clip_name)
|
||
clip_path: pathlib.Path | None = None
|
||
if artifact is not None:
|
||
try:
|
||
clip_path = source_path_from_item(
|
||
{"source": source_ref_from_artifact(artifact), "path": artifact.get("path", "")},
|
||
clip_name,
|
||
output_dir,
|
||
archive_sources,
|
||
)
|
||
except RuntimeError:
|
||
clip_path = None
|
||
if clip_path is None:
|
||
direct_clip_path = str(item.get("clipPath", "")).strip()
|
||
if direct_clip_path and pathlib.Path(direct_clip_path).exists():
|
||
clip_path = pathlib.Path(direct_clip_path)
|
||
if clip_path is None:
|
||
clip_path = materialize_clip_from_source(item, output_dir, archive_sources)
|
||
if clip_path is not None and clip_path.exists():
|
||
shutil.copy2(clip_path, package_dir / clip_name)
|
||
quality_payload = {
|
||
"clip_id": clip_id,
|
||
"source_video": item.get("sourceVideo", ""),
|
||
**(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {}),
|
||
}
|
||
annotation_payload = item.get("annotation") if isinstance(item.get("annotation"), dict) else {}
|
||
config_payload = package_config_payload(item, payload)
|
||
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_quality.json", quality_payload, output=output_binding(node_data))
|
||
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_annotation.json", annotation_payload, output=output_binding(node_data))
|
||
write_json_artifact(run_id, node_id, package_dir, f"{clip_id}_config.json", config_payload, output=output_binding(node_data))
|
||
package_files = [candidate for candidate in package_dir.rglob("*") if candidate.is_file()]
|
||
packages.append(
|
||
{
|
||
"packageName": package_name,
|
||
"clipId": clip_id,
|
||
"clipName": clip_name,
|
||
"sourceVideo": item.get("sourceVideo", ""),
|
||
"clipDuration": item.get("duration", 0),
|
||
"duration": item.get("duration", 0),
|
||
"startSecond": item.get("startSecond"),
|
||
"endSecond": item.get("endSecond"),
|
||
"collectionId": item.get("collectionId", ""),
|
||
"meta": item.get("meta", {}),
|
||
"source": item.get("source", {}) if isinstance(item.get("source"), dict) else {},
|
||
"fileCount": len(package_files),
|
||
"pipelineTags": ["quality_v1", "slicing_v1", "annotation_v1"],
|
||
"storagePath": str(package_dir),
|
||
"quality": quality_payload,
|
||
"annotation": annotation_payload,
|
||
}
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-packages",
|
||
"packageCount": len(packages),
|
||
"items": packages,
|
||
}
|
||
artifacts = [write_json_artifact(run_id, node_id, output_dir, "packages-manifest.json", manifest, output=output_binding(node_data))]
|
||
artifacts.extend(collect_output_artifacts(run_id, node_id, output_dir / "packages", node_data=node_data, output=output_binding(node_data)))
|
||
return artifacts
|
||
|
||
|
||
def builtin_audit_sampling(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
payload = first_payload_with_keys(inputs, "packageCount", "items")
|
||
if payload is None:
|
||
raise RuntimeError("抽检采样需要上游成品数据输入")
|
||
rules = parse_json_param(params, "RULES_JSON", [])
|
||
sampled_items = []
|
||
summaries = []
|
||
for rule in rules if isinstance(rules, list) else []:
|
||
if not isinstance(rule, dict):
|
||
continue
|
||
rule_id = str(rule.get("id", "rule")).strip() or "rule"
|
||
field_path = str(rule.get("field", "")).strip()
|
||
operator = str(rule.get("operator", "lt")).strip() or "lt"
|
||
value = rule.get("value")
|
||
percent = float(rule.get("samplePercent", 10) or 0)
|
||
matched = []
|
||
sampled = []
|
||
for item in payload.get("items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
if not compare_values(get_field_value(item, field_path), operator, value):
|
||
continue
|
||
matched.append(item)
|
||
package_name = str(item.get("packageName", "")).strip() or str(item.get("clipId", "")).strip()
|
||
if stable_sample_hit(f"{rule_id}:{package_name}", percent):
|
||
sampled.append(item)
|
||
sampled_items.append({**item, "sampleRuleId": rule_id})
|
||
summaries.append(
|
||
{
|
||
"id": rule_id,
|
||
"field": field_path,
|
||
"operator": operator,
|
||
"value": value,
|
||
"samplePercent": percent,
|
||
"matchedCount": len(matched),
|
||
"sampledCount": len(sampled),
|
||
}
|
||
)
|
||
manifest = {
|
||
"datasetType": "x-platform-audit-samples",
|
||
"rules": summaries,
|
||
"sampledCount": len(sampled_items),
|
||
"items": sampled_items,
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "audit-samples.json", manifest, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_condition_router(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
field_path = params.get("FIELD_PATH", "").strip()
|
||
operator = params.get("OPERATOR", "eq").strip().lower() or "eq"
|
||
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
|
||
rows = collect_items(payloads_from_inputs(inputs))
|
||
matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
|
||
unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)]
|
||
outputs = node_data.get("outputs", []) or [
|
||
{"id": "matched", "name": "matched", "type": "json"},
|
||
{"id": "unmatched", "name": "unmatched", "type": "json"},
|
||
]
|
||
return [
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json",
|
||
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)},
|
||
str(outputs[0].get("type", "json")),
|
||
output=outputs[0],
|
||
),
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json",
|
||
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)},
|
||
str(outputs[1].get("type", "json")),
|
||
output=outputs[1],
|
||
),
|
||
]
|
||
|
||
|
||
def builtin_lerobot_summary(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent)
|
||
payload = summarize_lerobot_root(root)
|
||
payload["sourcePath"] = str(source_path)
|
||
elif inputs:
|
||
payload = summarize_lerobot_from_inputs(inputs)
|
||
payload["sourcePath"] = ""
|
||
else:
|
||
raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录")
|
||
payload["datasetType"] = "lerobot"
|
||
return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_rosbag_summary(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
params = hyperparams(node_data)
|
||
source_path = resolve_source_path(params, inputs)
|
||
if source_path is not None:
|
||
if source_path.is_dir():
|
||
payload = {
|
||
"datasetType": "rosbag2",
|
||
"sourcePath": str(source_path),
|
||
"summary": summarize_directory_tree(source_path),
|
||
}
|
||
metadata_yaml = source_path / "metadata.yaml"
|
||
if metadata_yaml.exists():
|
||
payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"]
|
||
db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()]
|
||
if db3_files:
|
||
payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"]
|
||
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
||
payload = summarize_rosbag_file(source_path)
|
||
payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag")
|
||
payload["sourcePath"] = str(source_path)
|
||
else:
|
||
for artifact in inputs:
|
||
preview = artifact_preview_payload(artifact) or {}
|
||
if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}:
|
||
payload = {
|
||
"datasetType": preview.get("format", "rosbag"),
|
||
"sourcePath": str(artifact.get("path", "")),
|
||
"preview": preview,
|
||
}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
||
raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件")
|
||
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def builtin_generic(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_label: str,
|
||
node_data: dict[str, Any],
|
||
inputs: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
payloads = [load_artifact_payload(item) for item in inputs]
|
||
summary = {
|
||
"nodeLabel": node_label,
|
||
"inputCount": len(inputs),
|
||
"inputs": [summarize_json_like(item) for item in payloads],
|
||
"config": {
|
||
"bucket": node_data.get("bucket", ""),
|
||
"prefix": node_data.get("prefix", ""),
|
||
"pattern": node_data.get("pattern", ""),
|
||
},
|
||
}
|
||
outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}]
|
||
artifacts: list[dict[str, Any]] = []
|
||
for index, output in enumerate(outputs):
|
||
output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}"
|
||
filename = f"{output_name}.json"
|
||
payload = {**summary, "outputName": output_name}
|
||
artifacts.append(
|
||
write_json_artifact(
|
||
run_id,
|
||
node_id,
|
||
output_dir,
|
||
filename,
|
||
payload,
|
||
str(output.get("type", "json")),
|
||
output=output,
|
||
)
|
||
)
|
||
return artifacts
|
||
|
||
|
||
def execute_command(
|
||
run_id: str,
|
||
node_id: str,
|
||
output_dir: pathlib.Path,
|
||
node_data: dict[str, Any],
|
||
command: str,
|
||
) -> list[dict[str, Any]]:
|
||
completed = subprocess.run(
|
||
command,
|
||
cwd=output_dir,
|
||
shell=True,
|
||
text=True,
|
||
capture_output=True,
|
||
env=os.environ.copy(),
|
||
check=False,
|
||
)
|
||
for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)):
|
||
for line in content.splitlines():
|
||
if line.strip():
|
||
log(f"[operator-runner][command][{stream_name}] {line}")
|
||
if completed.returncode != 0:
|
||
raise RuntimeError(f"command failed with exit code {completed.returncode}")
|
||
artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data)
|
||
if artifacts:
|
||
return artifacts
|
||
payload = {"stdout": completed.stdout, "stderr": completed.stderr}
|
||
return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))]
|
||
|
||
|
||
def select_handler(node_label: str) -> str:
|
||
normalized = node_label.lower()
|
||
if any(token in normalized for token in ("mongodb", "mongo 写入", "mongo write", "写入 mongo")):
|
||
return "mongodb_write"
|
||
if any(token in normalized for token in ("x 平台数据集索引", "x-platform", "数据集索引", "dataset index")):
|
||
return "xplatform_index"
|
||
if any(token in normalized for token in ("采集目录校验", "目录校验", "validate collection")):
|
||
return "collection_validate"
|
||
if any(token in normalized for token in ("视频质检", "quality check", "quality")):
|
||
return "video_quality"
|
||
if any(token in normalized for token in ("数据过滤", "dataset filter", "video filter")):
|
||
return "dataset_filter"
|
||
if any(token in normalized for token in ("视频切片", "video slicing", "slice")):
|
||
return "video_slicing"
|
||
if any(token in normalized for token in ("视频标注", "annotation", "annotate")):
|
||
return "video_annotation"
|
||
if any(token in normalized for token in ("成品打包", "package output", "package result")):
|
||
return "result_package"
|
||
if any(token in normalized for token in ("抽检采样", "audit sample", "sampling")):
|
||
return "audit_sampling"
|
||
if "lerobot" in normalized:
|
||
return "lerobot"
|
||
if any(token in normalized for token in ("rosbag", "mcap", "ros bag")):
|
||
return "rosbag"
|
||
if any(token in normalized for token in ("csv 转 json", "csv转json", "csv to json", "tsv 转 json", "tsv转json", "tsv to json")):
|
||
return "csv_to_json"
|
||
if any(token in normalized for token in ("json 转 csv", "json转csv", "json to csv")):
|
||
return "json_to_csv"
|
||
if any(token in normalized for token in ("字段提取", "json提取", "json 提取", "extract")):
|
||
return "json_extract"
|
||
if any(token in normalized for token in ("json过滤", "json 过滤", "filter")):
|
||
return "json_filter"
|
||
if any(token in normalized for token in ("json映射", "json 映射", "rename", "映射重命名")):
|
||
return "json_map"
|
||
if any(token in normalized for token in ("模板渲染", "template")):
|
||
return "text_template"
|
||
if any(token in normalized for token in ("条件分流", "conditional", "router", "route")):
|
||
return "condition_router"
|
||
if any(token in normalized for token in ("目录扫描", "directory", "browse", "browse dir")):
|
||
return "directory_scan"
|
||
if any(token in normalized for token in ("merge", "合并")):
|
||
return "merge"
|
||
if any(token in normalized for token in ("intersect", "交集")):
|
||
return "intersection"
|
||
if any(token in normalized for token in ("distribution", "分配")):
|
||
return "distribution"
|
||
if any(token in normalized for token in ("upload", "export", "上传", "导出")):
|
||
return "upload"
|
||
if any(token in normalized for token in ("source", "scanner", "oss", "scan", "扫描", "数据源")):
|
||
return "source"
|
||
return "generic"
|
||
|
||
|
||
def main() -> int:
|
||
run_id = os.getenv("RUN_ID", "").strip()
|
||
workflow_id = os.getenv("WORKFLOW_ID", "").strip()
|
||
node_id = os.getenv("NODE_ID", "").strip()
|
||
node_label = os.getenv("NODE_LABEL", "").strip() or node_id
|
||
output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id)))
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
node_data = load_json_env("NODE_DATA_JSON", {})
|
||
inputs = load_input_artifacts()
|
||
started_at = now_iso()
|
||
start_cpu, start_memory = resource_snapshot()
|
||
|
||
logs: list[dict[str, Any]] = []
|
||
|
||
def append_log(level: str, message: str) -> None:
|
||
logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label})
|
||
log(f"[operator-runner][{level.lower()}] {message}")
|
||
|
||
append_log("INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir}")
|
||
append_log("INFO", "Preparing operator execution context")
|
||
|
||
params = hyperparams(node_data)
|
||
command = params.get("COMMAND") or params.get("SCRIPT")
|
||
artifacts: list[dict[str, Any]]
|
||
if command:
|
||
append_log("INFO", f"Executing explicit command: {command}")
|
||
artifacts = execute_command(run_id, node_id, output_dir, node_data, command)
|
||
else:
|
||
handler = select_handler(node_label)
|
||
append_log("INFO", f"Selected built-in handler: {handler}")
|
||
if handler == "source":
|
||
artifacts = builtin_source(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "merge":
|
||
artifacts = builtin_merge(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "intersection":
|
||
artifacts = builtin_intersection(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "distribution":
|
||
artifacts = builtin_distribution(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "json_extract":
|
||
artifacts = builtin_json_extract(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "json_filter":
|
||
artifacts = builtin_json_filter(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "json_map":
|
||
artifacts = builtin_json_map(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "mongodb_write":
|
||
artifacts = builtin_mongodb_write(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "text_template":
|
||
artifacts = builtin_text_template(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "csv_to_json":
|
||
artifacts = builtin_csv_to_json(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "json_to_csv":
|
||
artifacts = builtin_json_to_csv(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "directory_scan":
|
||
artifacts = builtin_directory_scan(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "xplatform_index":
|
||
artifacts = builtin_xplatform_index(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "collection_validate":
|
||
artifacts = builtin_collection_validate(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "video_quality":
|
||
artifacts = builtin_video_quality(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "dataset_filter":
|
||
artifacts = builtin_dataset_filter(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "video_slicing":
|
||
artifacts = builtin_video_slicing(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "video_annotation":
|
||
artifacts = builtin_video_annotation(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "result_package":
|
||
artifacts = builtin_result_package(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "audit_sampling":
|
||
artifacts = builtin_audit_sampling(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "condition_router":
|
||
artifacts = builtin_condition_router(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "lerobot":
|
||
artifacts = builtin_lerobot_summary(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "rosbag":
|
||
artifacts = builtin_rosbag_summary(run_id, node_id, output_dir, node_data, inputs)
|
||
elif handler == "upload":
|
||
artifacts = builtin_upload(run_id, node_id, output_dir, node_data, inputs)
|
||
else:
|
||
artifacts = builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs)
|
||
|
||
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
|
||
result = {
|
||
"logs": logs,
|
||
"artifacts": artifacts,
|
||
"resourceSeries": resource_series(start_cpu, start_memory, started_at),
|
||
}
|
||
stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip()
|
||
if stage_manifest_path:
|
||
write_json_file(stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
|
||
else:
|
||
write_json_file(default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
|
||
|
||
result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip()
|
||
if result_output_path:
|
||
written_path = write_json_file(result_output_path, result)
|
||
print(RESULT_PATH_PREFIX + str(written_path), flush=True)
|
||
return 0
|
||
|
||
default_result_path = write_json_file(default_result_output_path(output_dir), result)
|
||
print(RESULT_PATH_PREFIX + str(default_result_path), flush=True)
|
||
return 0
|
||
|
||
|
||
def tempfile_dir(run_id: str, node_id: str) -> str:
|
||
base = pathlib.Path("/tmp/droboticflow")
|
||
return str(base / (run_id or "run") / (node_id or "node"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
sys.exit(main())
|
||
except Exception as error: # pragma: no cover
|
||
log(f"[operator-runner][error] {error}")
|
||
sys.exit(1)
|