1471 lines
54 KiB
Python
1471 lines
54 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import fnmatch
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import pathlib
|
|
import base64
|
|
import csv
|
|
import re
|
|
import resource
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from typing import Any
|
|
|
|
|
|
RESULT_PREFIX = "DROBOTICFLOW_RESULT="
|
|
TEXT_FILE_SUFFIXES = {
|
|
".txt",
|
|
".log",
|
|
".md",
|
|
".yaml",
|
|
".yml",
|
|
".xml",
|
|
".html",
|
|
".htm",
|
|
}
|
|
TABULAR_FILE_SUFFIXES = {".csv": ",", ".tsv": "\t"}
|
|
IMAGE_FILE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
|
|
VIDEO_FILE_SUFFIXES = {".mp4", ".mov", ".avi", ".mkv", ".webm"}
|
|
AUDIO_FILE_SUFFIXES = {".wav", ".mp3", ".aac", ".m4a", ".ogg", ".flac"}
|
|
ROSBAG_FILE_SUFFIXES = {".bag", ".db3", ".mcap"}
|
|
PREVIEW_TEXT_LIMIT = 12_000
|
|
PREVIEW_TEXT_LINE_LIMIT = 240
|
|
PREVIEW_TABLE_ROW_LIMIT = 24
|
|
PREVIEW_FILE_LIST_LIMIT = 40
|
|
PREVIEW_INLINE_BINARY_LIMIT = 512 * 1024
|
|
|
|
|
|
def now_iso() -> str:
|
|
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def log(message: str) -> None:
|
|
print(message, flush=True)
|
|
|
|
|
|
def load_json_env(name: str, default: Any) -> Any:
|
|
value = os.getenv(name, "").strip()
|
|
if not value:
|
|
return default
|
|
return json.loads(value)
|
|
|
|
|
|
def resource_snapshot() -> tuple[float, int]:
|
|
usage = resource.getrusage(resource.RUSAGE_SELF)
|
|
cpu_seconds = usage.ru_utime + usage.ru_stime
|
|
if sys.platform == "darwin":
|
|
memory_mib = max(1, int(usage.ru_maxrss / (1024 * 1024)))
|
|
else:
|
|
memory_mib = max(1, int(usage.ru_maxrss / 1024))
|
|
return cpu_seconds, memory_mib
|
|
|
|
|
|
def resource_series(start_cpu: float, start_memory: int, started_at: str) -> list[dict[str, Any]]:
|
|
end_cpu, end_memory = resource_snapshot()
|
|
ended_at = now_iso()
|
|
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
|
|
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
|
|
elapsed = max((ended - started).total_seconds(), 0.001)
|
|
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
|
|
return [
|
|
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
|
|
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, "gpu": 0},
|
|
]
|
|
|
|
|
|
def format_size(num_bytes: int) -> str:
|
|
if num_bytes < 1024:
|
|
return f"{num_bytes} B"
|
|
if num_bytes < 1024 * 1024:
|
|
return f"{num_bytes / 1024:.1f} KiB"
|
|
if num_bytes < 1024 * 1024 * 1024:
|
|
return f"{num_bytes / (1024 * 1024):.1f} MiB"
|
|
return f"{num_bytes / (1024 * 1024 * 1024):.2f} GiB"
|
|
|
|
|
|
def sha256_file(path: pathlib.Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(65536), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def artifact_slug(value: str) -> str:
|
|
normalized = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value).strip("-")
|
|
return normalized or "artifact"
|
|
|
|
|
|
def infer_port_type(path: pathlib.Path, fallback: str = "file") -> str:
|
|
suffix = path.suffix.lower()
|
|
if suffix == ".json":
|
|
return "json"
|
|
if suffix in VIDEO_FILE_SUFFIXES:
|
|
return "video"
|
|
if suffix in IMAGE_FILE_SUFFIXES:
|
|
return "image"
|
|
return fallback
|
|
|
|
|
|
def clean_metadata(metadata: dict[str, Any]) -> dict[str, Any] | None:
|
|
cleaned = {key: value for key, value in metadata.items() if value not in ("", None, [], {})}
|
|
return cleaned or None
|
|
|
|
|
|
def load_payload_from_file(path: pathlib.Path) -> dict[str, Any] | None:
|
|
if not path.exists() or path.suffix.lower() != ".json":
|
|
return None
|
|
try:
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
value = json.load(handle)
|
|
except Exception:
|
|
return None
|
|
if isinstance(value, dict):
|
|
return value
|
|
if isinstance(value, list):
|
|
return {"items": value, "itemCount": len(value)}
|
|
return {"value": value}
|
|
|
|
|
|
def output_binding(node_data: dict[str, Any], index: int = 0, name: str = "") -> dict[str, Any] | None:
|
|
outputs = node_data.get("outputs", []) or []
|
|
if name:
|
|
for output in outputs:
|
|
if str(output.get("name", "")).strip() == name or str(output.get("id", "")).strip() == name:
|
|
return output
|
|
if 0 <= index < len(outputs):
|
|
return outputs[index]
|
|
return None
|
|
|
|
|
|
def read_text_sample(path: pathlib.Path) -> tuple[str, bool, int]:
|
|
with path.open("rb") as handle:
|
|
raw = handle.read(PREVIEW_TEXT_LIMIT + 1)
|
|
truncated = len(raw) > PREVIEW_TEXT_LIMIT
|
|
text = raw[:PREVIEW_TEXT_LIMIT].decode("utf-8", errors="replace")
|
|
lines = text.splitlines()
|
|
if len(lines) > PREVIEW_TEXT_LINE_LIMIT:
|
|
text = "\n".join(lines[:PREVIEW_TEXT_LINE_LIMIT])
|
|
truncated = True
|
|
line_count = max(1, text.count("\n") + 1) if text else 0
|
|
return text, truncated, line_count
|
|
|
|
|
|
def preview_text_payload(path: pathlib.Path, format_name: str) -> dict[str, Any]:
|
|
text, truncated, line_count = read_text_sample(path)
|
|
return {
|
|
"preview": {
|
|
"kind": "text",
|
|
"format": format_name,
|
|
"text": text,
|
|
"truncated": truncated,
|
|
"lineCount": line_count,
|
|
}
|
|
}
|
|
|
|
|
|
def preview_tabular_payload(path: pathlib.Path, delimiter: str, format_name: str) -> dict[str, Any]:
|
|
columns: list[str] = []
|
|
rows: list[dict[str, Any]] = []
|
|
total_rows = 0
|
|
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
|
|
reader = csv.reader(handle, delimiter=delimiter)
|
|
for index, row in enumerate(reader):
|
|
if index == 0:
|
|
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
|
|
continue
|
|
total_rows += 1
|
|
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
|
continue
|
|
width = max(len(columns), len(row))
|
|
row_payload: dict[str, Any] = {}
|
|
for column_index in range(width):
|
|
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
|
|
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
|
|
rows.append(row_payload)
|
|
return {
|
|
"preview": {
|
|
"kind": "table",
|
|
"format": format_name,
|
|
"columns": columns,
|
|
"rows": rows,
|
|
"rowCount": total_rows,
|
|
"truncated": total_rows > len(rows),
|
|
}
|
|
}
|
|
|
|
|
|
def load_tabular_rows(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Any]]]:
|
|
columns: list[str] = []
|
|
rows: list[dict[str, Any]] = []
|
|
with path.open("r", encoding="utf-8", errors="replace", newline="") as handle:
|
|
reader = csv.reader(handle, delimiter=delimiter)
|
|
for index, row in enumerate(reader):
|
|
if index == 0:
|
|
columns = [cell or f"column_{col_index + 1}" for col_index, cell in enumerate(row)]
|
|
continue
|
|
width = max(len(columns), len(row))
|
|
row_payload: dict[str, Any] = {}
|
|
for column_index in range(width):
|
|
column_name = columns[column_index] if column_index < len(columns) else f"column_{column_index + 1}"
|
|
row_payload[column_name] = row[column_index] if column_index < len(row) else ""
|
|
rows.append(row_payload)
|
|
return columns, rows
|
|
|
|
|
|
def preview_jsonl_payload(path: pathlib.Path) -> dict[str, Any]:
|
|
rows: list[Any] = []
|
|
line_count = 0
|
|
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
for raw_line in handle:
|
|
line = raw_line.strip()
|
|
if not line:
|
|
continue
|
|
line_count += 1
|
|
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
|
continue
|
|
try:
|
|
rows.append(json.loads(line))
|
|
except Exception:
|
|
rows.append({"value": line})
|
|
columns: list[str] = []
|
|
for row in rows:
|
|
if isinstance(row, dict):
|
|
for key in row.keys():
|
|
if key not in columns:
|
|
columns.append(key)
|
|
return {
|
|
"preview": {
|
|
"kind": "table",
|
|
"format": "jsonl",
|
|
"columns": columns,
|
|
"rows": rows,
|
|
"rowCount": line_count,
|
|
"truncated": line_count > len(rows),
|
|
}
|
|
}
|
|
|
|
|
|
def preview_image_payload(path: pathlib.Path, mime_type: str) -> dict[str, Any]:
|
|
payload: dict[str, Any] = {
|
|
"preview": {
|
|
"kind": "image",
|
|
"format": path.suffix.lower().lstrip(".") or "image",
|
|
}
|
|
}
|
|
size_bytes = path.stat().st_size if path.exists() else 0
|
|
if size_bytes and size_bytes <= PREVIEW_INLINE_BINARY_LIMIT and mime_type:
|
|
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
|
payload["preview"]["url"] = f"data:{mime_type};base64,{encoded}"
|
|
return payload
|
|
|
|
|
|
def sample_file_entries(paths: list[pathlib.Path], root: pathlib.Path) -> list[dict[str, Any]]:
|
|
items: list[dict[str, Any]] = []
|
|
for candidate in paths[:PREVIEW_FILE_LIST_LIMIT]:
|
|
try:
|
|
relative = candidate.relative_to(root).as_posix()
|
|
except ValueError:
|
|
relative = candidate.name
|
|
items.append({"path": relative, "sizeBytes": candidate.stat().st_size})
|
|
return items
|
|
|
|
|
|
def summarize_directory_tree(root: pathlib.Path) -> dict[str, Any]:
|
|
files = [candidate for candidate in sorted(root.rglob("*")) if candidate.is_file()]
|
|
extensions: dict[str, int] = {}
|
|
total_bytes = 0
|
|
for candidate in files:
|
|
extension = candidate.suffix.lower() or "<none>"
|
|
extensions[extension] = extensions.get(extension, 0) + 1
|
|
total_bytes += candidate.stat().st_size
|
|
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
|
|
return {
|
|
"root": str(root),
|
|
"fileCount": len(files),
|
|
"totalBytes": total_bytes,
|
|
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
|
|
"files": sample_file_entries(files, root),
|
|
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
|
|
}
|
|
|
|
|
|
def optional_json_file(path: pathlib.Path) -> Any:
|
|
if not path.exists():
|
|
return None
|
|
try:
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def optional_jsonl_rows(path: pathlib.Path) -> list[Any]:
|
|
rows: list[Any] = []
|
|
if not path.exists():
|
|
return rows
|
|
try:
|
|
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
for raw_line in handle:
|
|
line = raw_line.strip()
|
|
if not line:
|
|
continue
|
|
rows.append(json.loads(line))
|
|
if len(rows) >= PREVIEW_TABLE_ROW_LIMIT:
|
|
break
|
|
except Exception:
|
|
return []
|
|
return rows
|
|
|
|
|
|
def find_lerobot_root(path: pathlib.Path) -> pathlib.Path | None:
|
|
candidates = [path] if path.is_dir() else [path.parent]
|
|
candidates.extend(list((path if path.is_dir() else path.parent).parents))
|
|
for candidate in candidates:
|
|
meta_dir = candidate / "meta"
|
|
if meta_dir.is_dir() and any((meta_dir / name).exists() for name in ("info.json", "stats.json", "tasks.jsonl")):
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def summarize_lerobot_root(root: pathlib.Path) -> dict[str, Any]:
|
|
tree = summarize_directory_tree(root)
|
|
meta_dir = root / "meta"
|
|
info = optional_json_file(meta_dir / "info.json")
|
|
stats = optional_json_file(meta_dir / "stats.json")
|
|
tasks = optional_jsonl_rows(meta_dir / "tasks.jsonl")
|
|
episodes = optional_jsonl_rows(meta_dir / "episodes.jsonl")
|
|
parquet_files = [candidate for candidate in root.rglob("*.parquet") if candidate.is_file()]
|
|
video_files = [candidate for candidate in root.rglob("*") if candidate.is_file() and candidate.suffix.lower() in VIDEO_FILE_SUFFIXES]
|
|
return {
|
|
"preview": {
|
|
"kind": "dataset",
|
|
"format": "lerobot",
|
|
"root": str(root),
|
|
"summary": tree,
|
|
"info": info,
|
|
"stats": stats,
|
|
"tasks": tasks,
|
|
"episodes": episodes,
|
|
"parquetFiles": sample_file_entries(parquet_files, root),
|
|
"parquetCount": len(parquet_files),
|
|
"videoCount": len(video_files),
|
|
}
|
|
}
|
|
|
|
|
|
def sqlite_table_names(connection: sqlite3.Connection) -> list[str]:
|
|
rows = connection.execute(
|
|
"select name from sqlite_master where type='table' and name not like 'sqlite_%' order by name"
|
|
).fetchall()
|
|
return [str(row[0]) for row in rows]
|
|
|
|
|
|
def summarize_sqlite_payload(path: pathlib.Path) -> dict[str, Any]:
|
|
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
try:
|
|
tables = sqlite_table_names(connection)
|
|
table_summaries: list[dict[str, Any]] = []
|
|
for table_name in tables[:12]:
|
|
count = connection.execute(f"select count(*) from \"{table_name}\"").fetchone()[0]
|
|
table_summaries.append({"name": table_name, "rowCount": int(count)})
|
|
payload: dict[str, Any] = {
|
|
"preview": {
|
|
"kind": "dataset",
|
|
"format": "sqlite",
|
|
"tables": table_summaries,
|
|
"tableCount": len(tables),
|
|
}
|
|
}
|
|
if "topics" in tables and "messages" in tables:
|
|
topic_rows = connection.execute(
|
|
"select id, name, type, serialization_format from topics order by id"
|
|
).fetchall()
|
|
message_counts = {
|
|
int(topic_id): int(count)
|
|
for topic_id, count in connection.execute(
|
|
"select topic_id, count(*) from messages group by topic_id order by topic_id"
|
|
).fetchall()
|
|
}
|
|
payload["preview"]["format"] = "rosbag2"
|
|
payload["preview"]["topics"] = [
|
|
{
|
|
"id": int(topic_id),
|
|
"name": str(name),
|
|
"type": str(topic_type),
|
|
"serialization": str(serialization),
|
|
"messageCount": message_counts.get(int(topic_id), 0),
|
|
}
|
|
for topic_id, name, topic_type, serialization in topic_rows[:PREVIEW_TABLE_ROW_LIMIT]
|
|
]
|
|
payload["preview"]["messageCount"] = int(sum(message_counts.values()))
|
|
return payload
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def summarize_rosbag_file(path: pathlib.Path) -> dict[str, Any]:
|
|
suffix = path.suffix.lower()
|
|
if suffix == ".db3":
|
|
return summarize_sqlite_payload(path)
|
|
header = ""
|
|
with path.open("rb") as handle:
|
|
header = handle.read(32).decode("utf-8", errors="replace")
|
|
format_name = "mcap" if suffix == ".mcap" else "rosbag1"
|
|
return {
|
|
"preview": {
|
|
"kind": "dataset",
|
|
"format": format_name,
|
|
"header": header.strip(),
|
|
"note": "文件已识别,但深度解析依赖外部运行时;当前展示真实文件摘要和元数据。",
|
|
}
|
|
}
|
|
|
|
|
|
def resolve_source_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path | None:
|
|
direct = params.get("SOURCE_PATH", "").strip()
|
|
if direct and pathlib.Path(direct).exists():
|
|
return pathlib.Path(direct)
|
|
for artifact in inputs:
|
|
payload = artifact.get("data")
|
|
if isinstance(payload, dict):
|
|
candidate = payload.get("sourcePath")
|
|
if isinstance(candidate, str) and candidate and pathlib.Path(candidate).exists():
|
|
return pathlib.Path(candidate)
|
|
source_path = artifact.get("path")
|
|
if isinstance(source_path, str) and source_path and pathlib.Path(source_path).exists():
|
|
return pathlib.Path(source_path)
|
|
return None
|
|
|
|
|
|
def artifact_preview_payload(artifact: dict[str, Any]) -> dict[str, Any] | None:
|
|
data = artifact.get("data")
|
|
if not isinstance(data, dict):
|
|
return None
|
|
preview = data.get("preview")
|
|
if isinstance(preview, dict):
|
|
return preview
|
|
return None
|
|
|
|
|
|
def summarize_input_artifacts(inputs: list[dict[str, Any]]) -> dict[str, Any]:
|
|
extensions: dict[str, int] = {}
|
|
files: list[dict[str, Any]] = []
|
|
for artifact in inputs:
|
|
name = str(artifact.get("name", "") or artifact.get("path", "")).strip()
|
|
extension = pathlib.Path(name).suffix.lower() or "<none>"
|
|
extensions[extension] = extensions.get(extension, 0) + 1
|
|
files.append(
|
|
{
|
|
"path": str(artifact.get("path", "")).strip() or name,
|
|
"name": name,
|
|
"size": str(artifact.get("size", "")),
|
|
}
|
|
)
|
|
ranked_extensions = sorted(extensions.items(), key=lambda item: (-item[1], item[0]))
|
|
return {
|
|
"fileCount": len(files),
|
|
"topExtensions": [{"extension": extension, "count": count} for extension, count in ranked_extensions[:12]],
|
|
"files": files[:PREVIEW_FILE_LIST_LIMIT],
|
|
"truncated": len(files) > PREVIEW_FILE_LIST_LIMIT,
|
|
}
|
|
|
|
|
|
def infer_lerobot_root_from_inputs(inputs: list[dict[str, Any]]) -> str:
|
|
for artifact in inputs:
|
|
raw_path = str(artifact.get("path", "")).strip()
|
|
if not raw_path:
|
|
continue
|
|
parts = pathlib.PurePosixPath(raw_path).parts
|
|
for marker in ("meta", "data", "videos"):
|
|
if marker in parts:
|
|
index = parts.index(marker)
|
|
if index > 0:
|
|
return str(pathlib.PurePosixPath(*parts[:index]))
|
|
return ""
|
|
|
|
|
|
def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any]:
|
|
info = None
|
|
stats = None
|
|
tasks: list[Any] = []
|
|
episodes: list[Any] = []
|
|
parquet_files: list[dict[str, Any]] = []
|
|
video_count = 0
|
|
for artifact in inputs:
|
|
name = str(artifact.get("name", "")).strip()
|
|
path = str(artifact.get("path", "")).strip()
|
|
data = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
|
preview = artifact_preview_payload(artifact) or {}
|
|
if name == "info.json" and isinstance(data, dict):
|
|
info = data
|
|
elif name == "stats.json" and isinstance(data, dict):
|
|
stats = data
|
|
elif name.endswith("tasks.jsonl") and isinstance(preview.get("rows"), list):
|
|
tasks = preview["rows"]
|
|
elif name.endswith("episodes.jsonl") and isinstance(preview.get("rows"), list):
|
|
episodes = preview["rows"]
|
|
if name.endswith(".parquet"):
|
|
parquet_files.append({"path": path or name, "size": str(artifact.get("size", ""))})
|
|
if pathlib.Path(name).suffix.lower() in VIDEO_FILE_SUFFIXES or str(artifact.get("type", "")) == "video":
|
|
video_count += 1
|
|
return {
|
|
"preview": {
|
|
"kind": "dataset",
|
|
"format": "lerobot",
|
|
"root": infer_lerobot_root_from_inputs(inputs),
|
|
"summary": summarize_input_artifacts(inputs),
|
|
"info": info,
|
|
"stats": stats,
|
|
"tasks": tasks,
|
|
"episodes": episodes,
|
|
"parquetFiles": parquet_files[:PREVIEW_FILE_LIST_LIMIT],
|
|
"parquetCount": len(parquet_files),
|
|
"videoCount": video_count,
|
|
}
|
|
}
|
|
|
|
|
|
def extract_preview_data(path: pathlib.Path, artifact_type: str, mime_type: str) -> dict[str, Any] | None:
|
|
suffix = path.suffix.lower()
|
|
if suffix in TABULAR_FILE_SUFFIXES:
|
|
return preview_tabular_payload(path, TABULAR_FILE_SUFFIXES[suffix], suffix.lstrip("."))
|
|
if suffix == ".jsonl":
|
|
return preview_jsonl_payload(path)
|
|
if suffix in TEXT_FILE_SUFFIXES:
|
|
format_name = "html" if suffix in {".html", ".htm"} else suffix.lstrip(".")
|
|
return preview_text_payload(path, format_name)
|
|
if suffix in IMAGE_FILE_SUFFIXES:
|
|
return preview_image_payload(path, mime_type)
|
|
if suffix in ROSBAG_FILE_SUFFIXES:
|
|
return summarize_rosbag_file(path)
|
|
if suffix == ".pdf":
|
|
return {"preview": {"kind": "document", "format": "pdf"}}
|
|
if artifact_type == "video" or suffix in VIDEO_FILE_SUFFIXES:
|
|
return {"preview": {"kind": "video", "format": suffix.lstrip(".") or "video"}}
|
|
if suffix in AUDIO_FILE_SUFFIXES or mime_type.startswith("audio/"):
|
|
return {"preview": {"kind": "audio", "format": suffix.lstrip(".") or "audio"}}
|
|
return None
|
|
|
|
|
|
def build_artifact(
|
|
run_id: str,
|
|
node_id: str,
|
|
path: pathlib.Path,
|
|
artifact_type: str,
|
|
data: dict[str, Any] | None = None,
|
|
metadata: dict[str, Any] | None = None,
|
|
output: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
mime_type, _ = mimetypes.guess_type(path.name)
|
|
artifact_metadata = clean_metadata(
|
|
{
|
|
**(metadata or {}),
|
|
"sha256": sha256_file(path) if path.exists() else "",
|
|
"mimeType": mime_type or "",
|
|
}
|
|
)
|
|
size_bytes = path.stat().st_size if path.exists() else len(json.dumps(data or {}, ensure_ascii=False).encode("utf-8"))
|
|
artifact = {
|
|
"id": f"{run_id}-{node_id}-{artifact_slug(str(output.get('id', '')) if output else path.stem)}-{artifact_slug(path.name)}",
|
|
"runId": run_id,
|
|
"nodeId": node_id,
|
|
"name": path.name,
|
|
"type": artifact_type,
|
|
"size": format_size(size_bytes),
|
|
"path": str(path),
|
|
}
|
|
if output:
|
|
if str(output.get("id", "")).strip():
|
|
artifact["portId"] = str(output.get("id", "")).strip()
|
|
if str(output.get("name", "")).strip():
|
|
artifact["portName"] = str(output.get("name", "")).strip()
|
|
if artifact_metadata:
|
|
artifact["metadata"] = artifact_metadata
|
|
if data:
|
|
artifact["data"] = data
|
|
return artifact
|
|
|
|
|
|
def write_json_artifact(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
filename: str,
|
|
payload: dict[str, Any],
|
|
artifact_type: str = "json",
|
|
output: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
path = output_dir / filename
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
|
return build_artifact(run_id, node_id, path, artifact_type, data=payload, output=output)
|
|
|
|
|
|
def summarize_json_like(value: Any) -> dict[str, Any]:
|
|
if isinstance(value, dict):
|
|
return {"kind": "object", "keys": sorted(value.keys())}
|
|
if isinstance(value, list):
|
|
return {"kind": "list", "length": len(value)}
|
|
return {"kind": type(value).__name__, "value": value}
|
|
|
|
|
|
def load_artifact_payload(artifact: dict[str, Any]) -> dict[str, Any]:
|
|
data = artifact.get("data")
|
|
if isinstance(data, dict):
|
|
return data
|
|
path_value = artifact.get("path")
|
|
if isinstance(path_value, str) and path_value:
|
|
payload = load_payload_from_file(pathlib.Path(path_value))
|
|
if payload is not None:
|
|
return payload
|
|
summary = {
|
|
"name": artifact.get("name", ""),
|
|
"type": artifact.get("type", ""),
|
|
"path": artifact.get("path", ""),
|
|
"size": artifact.get("size", ""),
|
|
}
|
|
return {key: value for key, value in summary.items() if value}
|
|
|
|
|
|
def merge_dicts(values: list[dict[str, Any]]) -> dict[str, Any]:
|
|
merged: dict[str, Any] = {}
|
|
for value in values:
|
|
merged.update(value)
|
|
return merged
|
|
|
|
|
|
def list_source_entries(base_path: pathlib.Path, pattern: str) -> list[dict[str, Any]]:
|
|
entries: list[dict[str, Any]] = []
|
|
if not base_path.exists():
|
|
return entries
|
|
if base_path.is_file():
|
|
relative = base_path.name
|
|
if pattern and not fnmatch.fnmatch(relative, pattern):
|
|
return entries
|
|
return [
|
|
{
|
|
"path": relative,
|
|
"sizeBytes": base_path.stat().st_size,
|
|
"sha256": sha256_file(base_path),
|
|
}
|
|
]
|
|
for candidate in sorted(base_path.rglob("*")):
|
|
if candidate.is_dir():
|
|
continue
|
|
relative = candidate.relative_to(base_path).as_posix()
|
|
if pattern and not fnmatch.fnmatch(relative, pattern):
|
|
continue
|
|
entries.append(
|
|
{
|
|
"path": relative,
|
|
"sizeBytes": candidate.stat().st_size,
|
|
"sha256": sha256_file(candidate),
|
|
}
|
|
)
|
|
return entries
|
|
|
|
|
|
def collect_output_artifacts(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any] | None = None,
|
|
output: dict[str, Any] | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
artifacts: list[dict[str, Any]] = []
|
|
for path in sorted(output_dir.rglob("*")):
|
|
if path.is_dir():
|
|
continue
|
|
artifact_type = infer_port_type(path)
|
|
mime_type, _ = mimetypes.guess_type(path.name)
|
|
if artifact_type == "json":
|
|
payload = load_payload_from_file(path)
|
|
else:
|
|
payload = extract_preview_data(path, artifact_type, mime_type or "")
|
|
binding = output
|
|
if binding is None and node_data is not None:
|
|
binding = output_binding(node_data)
|
|
artifacts.append(build_artifact(run_id, node_id, path, artifact_type, data=payload, output=binding))
|
|
return artifacts
|
|
|
|
|
|
def hyperparams(node_data: dict[str, Any]) -> dict[str, str]:
|
|
values: dict[str, str] = {}
|
|
for item in node_data.get("hyperparams", []) or []:
|
|
key = str(item.get("key", "")).strip()
|
|
if not key:
|
|
continue
|
|
values[key] = str(item.get("value", ""))
|
|
return values
|
|
|
|
|
|
def parse_json_param(params: dict[str, str], key: str, default: Any) -> Any:
|
|
raw = params.get(key, "").strip()
|
|
if not raw:
|
|
return default
|
|
try:
|
|
return json.loads(raw)
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def payloads_from_inputs(inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [load_artifact_payload(item) for item in inputs]
|
|
|
|
|
|
def collect_items(payloads: list[dict[str, Any]]) -> list[Any]:
|
|
items: list[Any] = []
|
|
for payload in payloads:
|
|
if isinstance(payload.get("items"), list):
|
|
items.extend(payload["items"])
|
|
elif isinstance(payload.get("entries"), list):
|
|
items.extend(payload["entries"])
|
|
elif isinstance(payload.get("rows"), list):
|
|
items.extend(payload["rows"])
|
|
else:
|
|
items.append(payload)
|
|
return items
|
|
|
|
|
|
def get_field_value(value: Any, field_path: str) -> Any:
|
|
current = value
|
|
for token in [part for part in field_path.split(".") if part]:
|
|
if isinstance(current, dict):
|
|
current = current.get(token)
|
|
continue
|
|
if isinstance(current, list) and token.isdigit():
|
|
index = int(token)
|
|
if 0 <= index < len(current):
|
|
current = current[index]
|
|
continue
|
|
return None
|
|
return current
|
|
|
|
|
|
def compare_values(actual: Any, operator: str, expected: Any) -> bool:
|
|
if operator == "exists":
|
|
return actual is not None
|
|
if operator == "missing":
|
|
return actual is None
|
|
if operator == "contains":
|
|
if isinstance(actual, list):
|
|
return expected in actual
|
|
return str(expected) in str(actual)
|
|
if operator == "in":
|
|
if isinstance(expected, list):
|
|
return actual in expected
|
|
return str(actual) in str(expected)
|
|
if operator in {"gt", "gte", "lt", "lte"}:
|
|
try:
|
|
actual_number = float(actual)
|
|
expected_number = float(expected)
|
|
except Exception:
|
|
return False
|
|
if operator == "gt":
|
|
return actual_number > expected_number
|
|
if operator == "gte":
|
|
return actual_number >= expected_number
|
|
if operator == "lt":
|
|
return actual_number < expected_number
|
|
return actual_number <= expected_number
|
|
if operator == "ne":
|
|
return actual != expected
|
|
return actual == expected
|
|
|
|
|
|
def flatten_context(value: Any, prefix: str = "") -> dict[str, str]:
|
|
context: dict[str, str] = {}
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
next_prefix = f"{prefix}.{key}" if prefix else str(key)
|
|
context.update(flatten_context(item, next_prefix))
|
|
if prefix:
|
|
context[prefix] = json.dumps(value, ensure_ascii=False)
|
|
return context
|
|
if isinstance(value, list):
|
|
for index, item in enumerate(value[:8]):
|
|
next_prefix = f"{prefix}.{index}" if prefix else str(index)
|
|
context.update(flatten_context(item, next_prefix))
|
|
if prefix:
|
|
context[prefix] = json.dumps(value, ensure_ascii=False)
|
|
return context
|
|
if prefix:
|
|
context[prefix] = "" if value is None else str(value)
|
|
return context
|
|
|
|
|
|
def render_template_text(template_text: str, context: dict[str, str]) -> str:
|
|
rendered = template_text
|
|
for key, value in context.items():
|
|
rendered = rendered.replace("{{" + key + "}}", value)
|
|
safe_context = {key: value for key, value in context.items()}
|
|
|
|
class SafeDict(dict):
|
|
def __missing__(self, key: str) -> str:
|
|
return "{" + key + "}"
|
|
|
|
try:
|
|
rendered = rendered.format_map(SafeDict(safe_context))
|
|
except Exception:
|
|
pass
|
|
return rendered
|
|
|
|
|
|
def rows_from_payloads(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
for item in collect_items(payloads):
|
|
if isinstance(item, dict):
|
|
rows.append(item)
|
|
else:
|
|
rows.append({"value": item})
|
|
return rows
|
|
|
|
|
|
def write_text_artifact(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
filename: str,
|
|
content: str,
|
|
output: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
path = output_dir / filename
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content, encoding="utf-8")
|
|
mime_type, _ = mimetypes.guess_type(path.name)
|
|
preview = extract_preview_data(path, infer_port_type(path), mime_type or "")
|
|
return build_artifact(run_id, node_id, path, infer_port_type(path), data=preview, output=output)
|
|
|
|
|
|
def write_csv_artifact(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
filename: str,
|
|
rows: list[dict[str, Any]],
|
|
output: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
path = output_dir / filename
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
columns: list[str] = []
|
|
for row in rows:
|
|
for key in row.keys():
|
|
if key not in columns:
|
|
columns.append(key)
|
|
with path.open("w", encoding="utf-8", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=columns or ["value"])
|
|
writer.writeheader()
|
|
if rows:
|
|
writer.writerows(rows)
|
|
preview = preview_tabular_payload(path, ",", "csv")
|
|
return build_artifact(run_id, node_id, path, "file", data=preview, output=output)
|
|
|
|
|
|
def builtin_source(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
source_path = params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "").strip()
|
|
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
|
|
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
|
|
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
|
|
|
|
entries: list[dict[str, Any]] = []
|
|
if source_path:
|
|
entries = list_source_entries(pathlib.Path(source_path), pattern)
|
|
elif prefix and pathlib.Path(prefix).exists():
|
|
entries = list_source_entries(pathlib.Path(prefix), pattern)
|
|
elif params.get("SOURCE_JSON"):
|
|
parsed = json.loads(params["SOURCE_JSON"])
|
|
if isinstance(parsed, list):
|
|
entries = [{"value": item} for item in parsed]
|
|
else:
|
|
entries = [{"value": parsed}]
|
|
elif params.get("SOURCE_TEXT"):
|
|
entries = [{"text": params["SOURCE_TEXT"]}]
|
|
elif inputs:
|
|
entries = [load_artifact_payload(item) for item in inputs]
|
|
|
|
payload = {
|
|
"bucket": bucket,
|
|
"prefix": prefix,
|
|
"pattern": pattern,
|
|
"sourcePath": source_path,
|
|
"entries": entries,
|
|
}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "source-manifest.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_merge(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
payloads = payloads_from_inputs(inputs)
|
|
merged = merge_dicts(payloads)
|
|
merged["inputCount"] = len(payloads)
|
|
return [write_json_artifact(run_id, node_id, output_dir, "merged.json", merged, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_intersection(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
payloads = payloads_from_inputs(inputs)
|
|
if not payloads:
|
|
return [
|
|
write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"intersection.json",
|
|
{"keys": [], "inputCount": 0},
|
|
output=output_binding(node_data),
|
|
)
|
|
]
|
|
key_sets = [set(payload.keys()) for payload in payloads]
|
|
common_keys = sorted(set.intersection(*key_sets)) if key_sets else []
|
|
payload = {"keys": common_keys, "values": {key: payloads[0].get(key) for key in common_keys}}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "intersection.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_distribution(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
payloads = payloads_from_inputs(inputs)
|
|
items = collect_items(payloads)
|
|
outputs = node_data.get("outputs", []) or [{"name": "queue-a", "type": "json"}]
|
|
partitions: list[list[Any]] = [[] for _ in outputs]
|
|
for index, item in enumerate(items):
|
|
partitions[index % len(outputs)].append(item)
|
|
artifacts: list[dict[str, Any]] = []
|
|
for index, output in enumerate(outputs):
|
|
filename = f"{output.get('name', f'queue-{index}')}.json"
|
|
payload = {"items": partitions[index], "count": len(partitions[index])}
|
|
artifacts.append(
|
|
write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
filename,
|
|
payload,
|
|
str(output.get("type", "json")),
|
|
output=output,
|
|
)
|
|
)
|
|
return artifacts
|
|
|
|
|
|
def builtin_upload(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
copied_paths: list[str] = []
|
|
bundle_inputs: list[dict[str, Any]] = []
|
|
staging_dir = output_dir / "uploaded"
|
|
staging_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
for artifact in inputs:
|
|
payload = load_artifact_payload(artifact)
|
|
bundle_inputs.append(payload)
|
|
source_path = artifact.get("path")
|
|
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
|
|
source = pathlib.Path(source_path)
|
|
destination = staging_dir / source.name
|
|
if source.resolve() != destination.resolve():
|
|
shutil.copy2(source, destination)
|
|
copied_paths.append(str(destination))
|
|
|
|
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
|
|
artifacts = [write_json_artifact(run_id, node_id, output_dir, "uploaded-results.json", payload, output=output_binding(node_data))]
|
|
artifacts.extend(collect_output_artifacts(run_id, node_id, staging_dir))
|
|
return artifacts
|
|
|
|
|
|
def builtin_json_extract(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
field_path = params.get("FIELD_PATH", "").strip()
|
|
payloads = payloads_from_inputs(inputs)
|
|
payload = payloads[0] if payloads else {}
|
|
extracted = get_field_value(payload, field_path) if field_path else payload
|
|
result = {"fieldPath": field_path, "value": extracted}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "extracted.json", result, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_json_filter(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
field_path = params.get("FIELD_PATH", "").strip()
|
|
operator = params.get("OPERATOR", "eq").strip().lower() or "eq"
|
|
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
|
|
rows = collect_items(payloads_from_inputs(inputs))
|
|
filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
|
|
payload = {
|
|
"fieldPath": field_path,
|
|
"operator": operator,
|
|
"expected": expected,
|
|
"items": filtered,
|
|
"count": len(filtered),
|
|
}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "filtered.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_json_map(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
field_map = parse_json_param(params, "FIELD_MAP_JSON", {})
|
|
rows = rows_from_payloads(payloads_from_inputs(inputs))
|
|
mapped = [{str(field_map.get(key, key)): value for key, value in row.items()} for row in rows]
|
|
payload = {"fieldMap": field_map, "items": mapped, "count": len(mapped)}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "mapped.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_text_template(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
template_text = params.get("TEMPLATE_TEXT", "").strip()
|
|
output_name = params.get("OUTPUT_NAME", "").strip() or "rendered.txt"
|
|
payloads = payloads_from_inputs(inputs)
|
|
context = flatten_context(payloads[0] if payloads else {})
|
|
rendered = render_template_text(template_text, context)
|
|
binding = output_binding(node_data)
|
|
manifest = write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"rendered-template.json",
|
|
{"template": template_text, "context": context, "outputName": output_name},
|
|
output=binding,
|
|
)
|
|
return [manifest, write_text_artifact(run_id, node_id, output_dir, output_name, rendered, output=binding)]
|
|
|
|
|
|
def builtin_csv_to_json(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
source_path = resolve_source_path(params, inputs)
|
|
payload: dict[str, Any]
|
|
if source_path is not None and source_path.suffix.lower() in TABULAR_FILE_SUFFIXES:
|
|
delimiter = TABULAR_FILE_SUFFIXES[source_path.suffix.lower()]
|
|
columns, rows = load_tabular_rows(source_path, delimiter)
|
|
payload = {
|
|
"sourcePath": str(source_path),
|
|
"columns": columns,
|
|
"items": rows,
|
|
"rowCount": len(rows),
|
|
}
|
|
else:
|
|
for artifact in inputs:
|
|
preview = artifact_preview_payload(artifact) or {}
|
|
if preview.get("kind") == "table":
|
|
payload = {
|
|
"sourcePath": str(artifact.get("path", "")),
|
|
"columns": preview.get("columns", []),
|
|
"items": preview.get("rows", []),
|
|
"rowCount": preview.get("rowCount", len(preview.get("rows", []))),
|
|
"previewTruncated": preview.get("truncated", False),
|
|
}
|
|
return [
|
|
write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"table.json",
|
|
payload,
|
|
output=output_binding(node_data),
|
|
)
|
|
]
|
|
raise RuntimeError("CSV 转 JSON 需要 SOURCE_PATH 或上游 artifact 指向真实 csv/tsv 文件")
|
|
return [write_json_artifact(run_id, node_id, output_dir, "table.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_json_to_csv(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
rows = rows_from_payloads(payloads_from_inputs(inputs))
|
|
binding = output_binding(node_data)
|
|
summary = write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"csv-export-summary.json",
|
|
{"rowCount": len(rows), "columns": sorted({key for row in rows for key in row.keys()})},
|
|
output=binding,
|
|
)
|
|
return [summary, write_csv_artifact(run_id, node_id, output_dir, "rows.csv", rows, output=binding)]
|
|
|
|
|
|
def builtin_directory_scan(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
source_path = resolve_source_path(params, inputs)
|
|
if source_path is not None:
|
|
root = source_path if source_path.is_dir() else source_path.parent
|
|
payload = {
|
|
"sourcePath": str(source_path),
|
|
"scanRoot": str(root),
|
|
"summary": summarize_directory_tree(root),
|
|
}
|
|
elif inputs:
|
|
payload = {
|
|
"sourcePath": "",
|
|
"scanRoot": "",
|
|
"summary": summarize_input_artifacts(inputs),
|
|
}
|
|
else:
|
|
raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录")
|
|
return [write_json_artifact(run_id, node_id, output_dir, "directory-scan.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_condition_router(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
field_path = params.get("FIELD_PATH", "").strip()
|
|
operator = params.get("OPERATOR", "eq").strip().lower() or "eq"
|
|
expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", ""))
|
|
rows = collect_items(payloads_from_inputs(inputs))
|
|
matched = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)]
|
|
unmatched = [row for row in rows if not compare_values(get_field_value(row, field_path), operator, expected)]
|
|
outputs = node_data.get("outputs", []) or [
|
|
{"id": "matched", "name": "matched", "type": "json"},
|
|
{"id": "unmatched", "name": "unmatched", "type": "json"},
|
|
]
|
|
return [
|
|
write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
f"{str(outputs[0].get('name', 'matched')).strip() or 'matched'}.json",
|
|
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": matched, "count": len(matched)},
|
|
str(outputs[0].get("type", "json")),
|
|
output=outputs[0],
|
|
),
|
|
write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
f"{str(outputs[1].get('name', 'unmatched')).strip() or 'unmatched'}.json",
|
|
{"fieldPath": field_path, "operator": operator, "expected": expected, "items": unmatched, "count": len(unmatched)},
|
|
str(outputs[1].get("type", "json")),
|
|
output=outputs[1],
|
|
),
|
|
]
|
|
|
|
|
|
def builtin_lerobot_summary(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
source_path = resolve_source_path(params, inputs)
|
|
if source_path is not None:
|
|
root = find_lerobot_root(source_path) or (source_path if source_path.is_dir() else source_path.parent)
|
|
payload = summarize_lerobot_root(root)
|
|
payload["sourcePath"] = str(source_path)
|
|
elif inputs:
|
|
payload = summarize_lerobot_from_inputs(inputs)
|
|
payload["sourcePath"] = ""
|
|
else:
|
|
raise RuntimeError("LeRobot 数据集概览需要 SOURCE_PATH 或上游 artifact 指向真实数据集目录")
|
|
payload["datasetType"] = "lerobot"
|
|
return [write_json_artifact(run_id, node_id, output_dir, "lerobot-summary.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_rosbag_summary(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
params = hyperparams(node_data)
|
|
source_path = resolve_source_path(params, inputs)
|
|
if source_path is not None:
|
|
if source_path.is_dir():
|
|
payload = {
|
|
"datasetType": "rosbag2",
|
|
"sourcePath": str(source_path),
|
|
"summary": summarize_directory_tree(source_path),
|
|
}
|
|
metadata_yaml = source_path / "metadata.yaml"
|
|
if metadata_yaml.exists():
|
|
payload["metadata"] = preview_text_payload(metadata_yaml, "yaml")["preview"]
|
|
db3_files = [candidate for candidate in source_path.rglob("*.db3") if candidate.is_file()]
|
|
if db3_files:
|
|
payload["databasePreview"] = summarize_sqlite_payload(db3_files[0])["preview"]
|
|
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
|
payload = summarize_rosbag_file(source_path)
|
|
payload["datasetType"] = payload.get("preview", {}).get("format", "rosbag")
|
|
payload["sourcePath"] = str(source_path)
|
|
else:
|
|
for artifact in inputs:
|
|
preview = artifact_preview_payload(artifact) or {}
|
|
if preview.get("kind") == "dataset" and preview.get("format") in {"rosbag2", "rosbag1", "mcap", "sqlite"}:
|
|
payload = {
|
|
"datasetType": preview.get("format", "rosbag"),
|
|
"sourcePath": str(artifact.get("path", "")),
|
|
"preview": preview,
|
|
}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
|
raise RuntimeError("ROS bag 概览需要 SOURCE_PATH 或上游 artifact 指向真实 bag/db3/mcap 文件")
|
|
return [write_json_artifact(run_id, node_id, output_dir, "rosbag-summary.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def builtin_generic(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_label: str,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
payloads = [load_artifact_payload(item) for item in inputs]
|
|
summary = {
|
|
"nodeLabel": node_label,
|
|
"inputCount": len(inputs),
|
|
"inputs": [summarize_json_like(item) for item in payloads],
|
|
"config": {
|
|
"bucket": node_data.get("bucket", ""),
|
|
"prefix": node_data.get("prefix", ""),
|
|
"pattern": node_data.get("pattern", ""),
|
|
},
|
|
}
|
|
outputs = node_data.get("outputs", []) or [{"name": "result", "type": "json"}]
|
|
artifacts: list[dict[str, Any]] = []
|
|
for index, output in enumerate(outputs):
|
|
output_name = str(output.get("name", f"output-{index}")).strip() or f"output-{index}"
|
|
filename = f"{output_name}.json"
|
|
payload = {**summary, "outputName": output_name}
|
|
artifacts.append(
|
|
write_json_artifact(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
filename,
|
|
payload,
|
|
str(output.get("type", "json")),
|
|
output=output,
|
|
)
|
|
)
|
|
return artifacts
|
|
|
|
|
|
def execute_command(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
node_data: dict[str, Any],
|
|
command: str,
|
|
) -> list[dict[str, Any]]:
|
|
completed = subprocess.run(
|
|
command,
|
|
cwd=output_dir,
|
|
shell=True,
|
|
text=True,
|
|
capture_output=True,
|
|
env=os.environ.copy(),
|
|
check=False,
|
|
)
|
|
for stream_name, content in (("stdout", completed.stdout), ("stderr", completed.stderr)):
|
|
for line in content.splitlines():
|
|
if line.strip():
|
|
log(f"[operator-runner][command][{stream_name}] {line}")
|
|
if completed.returncode != 0:
|
|
raise RuntimeError(f"command failed with exit code {completed.returncode}")
|
|
artifacts = collect_output_artifacts(run_id, node_id, output_dir, node_data=node_data)
|
|
if artifacts:
|
|
return artifacts
|
|
payload = {"stdout": completed.stdout, "stderr": completed.stderr}
|
|
return [write_json_artifact(run_id, node_id, output_dir, "command-output.json", payload, output=output_binding(node_data))]
|
|
|
|
|
|
def select_handler(node_label: str) -> str:
|
|
normalized = node_label.lower()
|
|
if "lerobot" in normalized:
|
|
return "lerobot"
|
|
if any(token in normalized for token in ("rosbag", "mcap", "ros bag")):
|
|
return "rosbag"
|
|
if any(token in normalized for token in ("csv 转 json", "csv转json", "csv to json", "tsv 转 json", "tsv转json", "tsv to json")):
|
|
return "csv_to_json"
|
|
if any(token in normalized for token in ("json 转 csv", "json转csv", "json to csv")):
|
|
return "json_to_csv"
|
|
if any(token in normalized for token in ("字段提取", "json提取", "json 提取", "extract")):
|
|
return "json_extract"
|
|
if any(token in normalized for token in ("json过滤", "json 过滤", "filter")):
|
|
return "json_filter"
|
|
if any(token in normalized for token in ("json映射", "json 映射", "rename", "映射重命名")):
|
|
return "json_map"
|
|
if any(token in normalized for token in ("模板渲染", "template")):
|
|
return "text_template"
|
|
if any(token in normalized for token in ("条件分流", "conditional", "router", "route")):
|
|
return "condition_router"
|
|
if any(token in normalized for token in ("目录扫描", "directory", "browse", "browse dir")):
|
|
return "directory_scan"
|
|
if any(token in normalized for token in ("merge", "合并")):
|
|
return "merge"
|
|
if any(token in normalized for token in ("intersect", "交集")):
|
|
return "intersection"
|
|
if any(token in normalized for token in ("distribution", "分配")):
|
|
return "distribution"
|
|
if any(token in normalized for token in ("upload", "export", "上传", "导出")):
|
|
return "upload"
|
|
if any(token in normalized for token in ("source", "scanner", "oss", "scan", "扫描")):
|
|
return "source"
|
|
return "generic"
|
|
|
|
|
|
def main() -> int:
|
|
run_id = os.getenv("RUN_ID", "").strip()
|
|
workflow_id = os.getenv("WORKFLOW_ID", "").strip()
|
|
node_id = os.getenv("NODE_ID", "").strip()
|
|
node_label = os.getenv("NODE_LABEL", "").strip() or node_id
|
|
output_dir = pathlib.Path(os.getenv("OUTPUT_DIR", tempfile_dir(run_id, node_id)))
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
node_data = load_json_env("NODE_DATA_JSON", {})
|
|
inputs = load_json_env("INPUT_ARTIFACTS_JSON", [])
|
|
started_at = now_iso()
|
|
start_cpu, start_memory = resource_snapshot()
|
|
|
|
logs: list[dict[str, Any]] = []
|
|
|
|
def append_log(level: str, message: str) -> None:
|
|
logs.append({"timestamp": now_iso(), "level": level, "message": message, "operator": node_label})
|
|
log(f"[operator-runner][{level.lower()}] {message}")
|
|
|
|
append_log("INFO", f"run={run_id} workflow={workflow_id} node={node_id} output_dir={output_dir}")
|
|
append_log("INFO", "Preparing operator execution context")
|
|
|
|
params = hyperparams(node_data)
|
|
command = params.get("COMMAND") or params.get("SCRIPT")
|
|
artifacts: list[dict[str, Any]]
|
|
if command:
|
|
append_log("INFO", f"Executing explicit command: {command}")
|
|
artifacts = execute_command(run_id, node_id, output_dir, node_data, command)
|
|
else:
|
|
handler = select_handler(node_label)
|
|
append_log("INFO", f"Selected built-in handler: {handler}")
|
|
if handler == "source":
|
|
artifacts = builtin_source(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "merge":
|
|
artifacts = builtin_merge(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "intersection":
|
|
artifacts = builtin_intersection(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "distribution":
|
|
artifacts = builtin_distribution(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "json_extract":
|
|
artifacts = builtin_json_extract(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "json_filter":
|
|
artifacts = builtin_json_filter(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "json_map":
|
|
artifacts = builtin_json_map(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "text_template":
|
|
artifacts = builtin_text_template(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "csv_to_json":
|
|
artifacts = builtin_csv_to_json(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "json_to_csv":
|
|
artifacts = builtin_json_to_csv(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "directory_scan":
|
|
artifacts = builtin_directory_scan(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "condition_router":
|
|
artifacts = builtin_condition_router(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "lerobot":
|
|
artifacts = builtin_lerobot_summary(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "rosbag":
|
|
artifacts = builtin_rosbag_summary(run_id, node_id, output_dir, node_data, inputs)
|
|
elif handler == "upload":
|
|
artifacts = builtin_upload(run_id, node_id, output_dir, node_data, inputs)
|
|
else:
|
|
artifacts = builtin_generic(run_id, node_id, output_dir, node_label, node_data, inputs)
|
|
|
|
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
|
|
result = {
|
|
"logs": logs,
|
|
"artifacts": artifacts,
|
|
"resourceSeries": resource_series(start_cpu, start_memory, started_at),
|
|
}
|
|
print(RESULT_PREFIX + json.dumps(result, ensure_ascii=False), flush=True)
|
|
return 0
|
|
|
|
|
|
def tempfile_dir(run_id: str, node_id: str) -> str:
|
|
base = pathlib.Path("/tmp/droboticflow")
|
|
return str(base / (run_id or "run") / (node_id or "node"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except Exception as error: # pragma: no cover
|
|
log(f"[operator-runner][error] {error}")
|
|
sys.exit(1)
|