droboticflow/scripts/ego_workflows.py
eust-w 6b3ed706c4 🐛 Correct ego CPU action checks and split report data flow
Validate the dataset's 78-dimensional actions, export split records instead of the leakage report, and render selected training episodes with supported template fields. Add regression coverage for record round-tripping, route counts, and empty reports.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-10-28 15:25:09 +08:00

1161 lines
55 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Inspect an egocentric LeRobot dataset on BOS and build processing workflows for it.
Usage:
python3 scripts/ego_workflows.py inspect
python3 scripts/ego_workflows.py create [KEY ...] [--run]
python3 scripts/ego_workflows.py status [KEY ...] [--wait]
Login: DROBOTICFLOW_EMAIL / DROBOTICFLOW_PASSWORD, otherwise prompted. The
session cookie is kept in output/ego-workflows/ so later steps reuse it.
"""
from __future__ import annotations
import argparse
import fnmatch
import getpass
import http.client
import http.cookiejar
import json
import os
import pathlib
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
ROOT = pathlib.Path(__file__).resolve().parents[1]
STATE_DIR = ROOT / "output" / "ego-workflows"
COOKIE_FILE = STATE_DIR / "cookies.txt"
API_BASE_URL = os.environ.get("API_BASE_URL", "http://120.48.164.19").rstrip("/")
BUCKET = os.environ.get("EGO_BUCKET", "testxyz")
DATASET_PREFIX = os.environ.get("EGO_DATASET_PREFIX", "digua_test/R16to9_10_sample").strip("/")
CAMERA_KEY = os.environ.get("EGO_CAMERA_KEY", "observation.images.cam_head_ego_left")
PROJECT_NAME = os.environ.get("EGO_PROJECT_NAME", "Ego 数据处理")
CONNECTION_ID = os.environ.get("EGO_CONNECTION_ID", "")
class Api:
def __init__(self) -> None:
STATE_DIR.mkdir(parents=True, exist_ok=True)
self.jar = http.cookiejar.MozillaCookieJar(str(COOKIE_FILE))
if COOKIE_FILE.exists():
self.jar.load(ignore_discard=True, ignore_expires=True)
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.jar))
def _csrf(self) -> str:
for cookie in self.jar:
if cookie.name == "droboticflow_csrf":
return cookie.value or ""
return ""
def request(self, method: str, path: str, body: Any = None, query: dict[str, Any] | None = None) -> Any:
url = API_BASE_URL + path
if query:
url += "?" + urllib.parse.urlencode({k: v for k, v in query.items() if v not in (None, "")})
data = None
headers = {"Accept": "application/json"}
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
headers["Content-Type"] = "application/json"
if method not in {"GET", "HEAD"} and self._csrf():
headers["X-Droboticflow-CSRF-Token"] = self._csrf()
request = urllib.request.Request(url, data=data, method=method, headers=headers)
try:
with self.opener.open(request, timeout=120) as response:
try:
raw = response.read()
except http.client.IncompleteRead as error:
raw = error.partial # the stop endpoints sometimes cut the chunked body short
except urllib.error.HTTPError as error:
detail = error.read().decode("utf-8", "replace")[:800]
raise RuntimeError(f"{method} {path} -> HTTP {error.code}: {detail}") from None
finally:
self.jar.save(ignore_discard=True, ignore_expires=True)
if not raw:
return None
try:
return json.loads(raw)
except json.JSONDecodeError:
return raw.decode("utf-8", "replace")
def ensure_login(self) -> dict[str, Any]:
try:
me = self.request("GET", "/api/auth/me")
if isinstance(me, dict) and (me.get("user") or me.get("id")):
return me
except RuntimeError:
pass
email = os.environ.get("DROBOTICFLOW_EMAIL") or input("Droboticflow 登录邮箱: ").strip()
password = os.environ.get("DROBOTICFLOW_PASSWORD") or getpass.getpass("密码: ")
self.request("POST", "/api/auth/login", {"email": email, "password": password})
return self.request("GET", "/api/auth/me")
def save(name: str, payload: Any) -> pathlib.Path:
path = STATE_DIR / name
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def resolve_connection_id(api: Api) -> str:
"""Pick the storage connection that exposes BUCKET (the org default may be unconfigured)."""
global CONNECTION_ID
if CONNECTION_ID:
return CONNECTION_ID
connections = api.request("GET", "/api/storage/connections")
items = connections.get("items", connections) if isinstance(connections, dict) else connections
for connection in items or []:
if connection.get("bucket") == BUCKET:
CONNECTION_ID = connection["id"]
return CONNECTION_ID
for connection in items or []:
overview = api.request("GET", "/api/object-storage", query={"connectionId": connection["id"]})
if BUCKET in [bucket.get("name") for bucket in overview.get("buckets") or []]:
CONNECTION_ID = connection["id"]
return CONNECTION_ID
raise RuntimeError(f"no storage connection exposes bucket {BUCKET}")
def list_directory(api: Api, prefix: str) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
marker = ""
while True:
page = api.request(
"GET",
"/api/object-storage/objects",
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": prefix, "marker": marker, "limit": 1000},
)
items.extend(page.get("items") or [])
marker = page.get("nextMarker") or ""
if not page.get("hasMore") or not marker:
return items
def list_objects(api: Api, prefix: str, limit: int = 20000) -> list[dict[str, Any]]:
"""Walk the directory-style listing into a flat list of files."""
files: list[dict[str, Any]] = []
pending = [prefix.rstrip("/") + "/"]
while pending and len(files) < limit:
current = pending.pop(0)
for item in list_directory(api, current):
if item.get("kind") == "directory":
pending.append(str(item["key"]).rstrip("/") + "/")
else:
files.append(item)
return files
def read_text_object(api: Api, key: str) -> str:
content = api.request(
"GET", "/api/object-storage/content", query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "key": key}
)
return content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
def cmd_inspect(api: Api) -> None:
me = api.ensure_login()
print("logged in:", json.dumps(me, ensure_ascii=False)[:200])
objects = list_objects(api, DATASET_PREFIX + "/")
keys = [str(item.get("key") or item.get("name") or "") for item in objects]
print(f"objects under bos://{BUCKET}/{DATASET_PREFIX}/: {len(keys)}")
meta: dict[str, Any] = {}
for key in keys:
if key.endswith((".json", ".jsonl")) and "/meta/" in f"/{key}" and len(meta) < 12:
try:
meta[key] = read_text_object(api, key)[:20000]
except RuntimeError as error:
meta[key] = f"ERROR: {error}"
path = save(
"inspect.json",
{"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": DATASET_PREFIX, "objects": objects, "meta": meta},
)
dirs: dict[str, int] = {}
for key in keys:
rel = key[len(DATASET_PREFIX) + 1 :] if key.startswith(DATASET_PREFIX + "/") else key
top = "/".join(rel.split("/")[:-1]) or "."
dirs[top] = dirs.get(top, 0) + 1
for name, count in sorted(dirs.items()):
print(f" {name}/ ({count} files)")
print("saved:", path)
# --- workflow definitions -------------------------------------------------
DATASET_URI = f"bos://{BUCKET}/{DATASET_PREFIX}"
OUTPUT_URI = os.environ.get("EGO_OUTPUT_URI", f"bos://{BUCKET}/{DATASET_PREFIX}_processed").rstrip("/")
LEFT_VIDEOS = f"videos/chunk-000/{CAMERA_KEY}"
SAMPLE_EPISODE = os.environ.get("EGO_SAMPLE_EPISODE", "episode_000000")
QC_GLOB = os.environ.get("EGO_QC_GLOB", "*.mp4")
CPU_VIDEO_GLOB = os.environ.get("EGO_CPU_VIDEO_GLOB", "episode_00000[05].mp4")
# Proven egodex pipeline (SAM3.1 v6 -> hand removal v4 -> upload) whose node settings are cloned.
# Validator images from completed acceptance runs; the live catalog's `:builtin` digests never start.
SCHEMA_TEMPLATE = "workflow-23533841935b7e283183138b19825b0f#op-lerobot-dataset-validate"
METADATA_TEMPLATE = "workflow-602cefb887ee6c1f189c89e395f82b44#op-lerobot-metadata-consistency-validate"
ALIGNMENT_TEMPLATE = "workflow-602cefb887ee6c1f189c89e395f82b44#op-lerobot-video-alignment-validate"
# Stable staging dir on the workspace PVC for operators that only read local paths (SAM3, hand removal).
STAGING_ROOT = f"/workspace/ego-staging/{DATASET_PREFIX}"
SAMPLE_VIDEO_PATH = f"{STAGING_ROOT}/{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4"
MASK_DIR = f"/workspace/ego-staging/masks/{DATASET_PREFIX}/{CAMERA_KEY}"
HAND_REMOVAL_TEMPLATE = os.environ.get("EGO_HAND_REMOVAL_TEMPLATE", "workflow-a77bd54acf37717a2a72339021182072")
# Each node: (node id, operator id, label, hyperparam overrides, BOS source pattern or None).
# Edges connect each operator's first output to the next operator's first input.
WORKFLOWS: dict[str, dict[str, Any]] = {
"smoke": {
"name": "Ego 冒烟-元数据读取",
"description": f"读取 {DATASET_URI} 的 meta/ 与 data/(不含视频),生成 LeRobot 数据集概览,用于验证 BOS 数据接入链路。",
"nodes": [
("source", "operator-builtin-real-source", "Ego 元数据与 Parquet", {}, "[md][ea]ta/*"),
("summary", "operator-builtin-lerobot-summary", "LeRobot 数据集概览", {}, None),
],
"edges": [("source", "summary")],
},
# Each check is its own two-node workflow (stage -> one validator). On the current POC
# worker, a third node in a chain or parallel branches stall before their Kubernetes Job
# is created; stage -> one consumer runs reliably. Staged files are reused across runs.
**{
key: {
"name": f"Ego 数据集校验·{title}",
"description": f"把 {DATASET_URI} 的 meta/、data/、videos/ 暂存到工作区 {STAGING_ROOT}(已暂存的文件复用),再执行{title}。",
"nodes": [
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
(node, operator, title, {"SOURCE_PATH": STAGING_ROOT, **overrides}, None),
],
"edges": [("stage", node, "out-1", "in-1")],
"inputs": [{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")}],
}
for key, node, title, operator, overrides in [
("integrity", "summary", "LeRobot 数据集概览", "operator-builtin-lerobot-summary", {}),
(
"integrity_schema",
"schema",
"Parquet 结构与索引校验",
"operator-builtin-lerobot-dataset-validate",
{"REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state"},
),
(
"integrity_metadata",
"metadata",
"元数据一致性校验",
"operator-builtin-lerobot-metadata-consistency-validate",
{"REQUIRED_FEATURES": f"action,observation.state,left_kp3d,right_kp3d,{CAMERA_KEY}"},
),
(
"integrity_alignment",
"alignment",
"视频帧对齐校验",
"operator-builtin-lerobot-video-alignment-validate",
{"FRAME_TOLERANCE": "1", "MAX_VIDEOS": "20"},
),
]
},
"left_qc": {
"name": "Ego 左目视频质检",
"description": (
f"对 {LEFT_VIDEOS} 的全部视频计算清晰度、抖动与无手率,并按阈值判定通过/不通过。"
"视频清单在每次运行时作为工作流输入传入(含 1 小时有效的 BOS 预签名下载地址)。"
),
# No separate filter node: video-quality already marks each item passed/failed, and on
# the current worker a node consuming its externalized dataRef output never launches.
"nodes": [
("quality", "operator-builtin-video-quality", "视频质检(含通过/不通过判定)", {}, None),
],
"edges": [],
"inputs": [
{"id": "videos", "name": "左目视频清单", "node": "quality", "port": "in-1", "items": (LEFT_VIDEOS, QC_GLOB)},
],
},
"wilor": {
"name": f"Ego 手部 3D 姿态估计(WiLoR,{SAMPLE_EPISODE})",
"description": (
f"对左目 {SAMPLE_EPISODE}.mp4 抽帧调用 WiLoR 估计双手 3D 姿态,结果上传到 {OUTPUT_URI}/wilor/。"
"INPUT_URL 在每次运行前刷新为 1 小时有效的 BOS 预签名地址。"
),
"nodes": [
(
"source",
"operator-builtin-real-source",
"Ego 视频引用",
{"DATA_SOURCE": "INLINE", "SOURCE_JSON": json.dumps({"video": f"{DATASET_URI}/{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4"})},
None,
),
(
"wilor",
"operator-builtin-wilor-hand-pose",
"WiLoR 3D 手部姿态",
{
"INPUT_URL": f"@url:{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4",
"INPUT_PATH": "",
"MEDIA_KIND": "video",
"FRAME_STRIDE": "3",
"MAX_FRAMES": "120",
},
None,
),
("upload", "operator-builtin-upload", "结果上传 BOS", {"OUTPUT_PATH": f"{OUTPUT_URI}/wilor/"}, "@output"),
],
"edges": [("source", "wilor"), ("wilor", "upload")],
},
# Hand removal is split in two for the same worker limitation. SAM3 and the hand-removal
# image only read local paths (artifacts arrive as bos:// refs), so the video and the masks
# go through stable workspace paths. Results are persisted by the worker's artifact storage.
"hand_removal": {
"name": f"Ego 去手①SAM3.1 手与手臂分割({SAMPLE_EPISODE})",
"description": f"把左目 {SAMPLE_EPISODE}.mp4 暂存到 {STAGING_ROOT},用 SAM3.1 分割手与手臂,掩码汇总到 {MASK_DIR}。",
"nodes": [
("source", "@stage", f"左目视频 {SAMPLE_EPISODE} 暂存", {"MODE": "tree", "SUBDIR": f"{STAGING_ROOT}/{LEFT_VIDEOS}"}, None),
("sam3", f"clone:{HAND_REMOVAL_TEMPLATE}#node-sam3-1", "SAM3.1 手与手臂分割", {"VIDEO_PATH": SAMPLE_VIDEO_PATH}, None),
("collect", "@stage", "SAM3 掩码汇总到工作区", {"MODE": "collect", "SOURCE_NODE": "node-sam3", "SUBDIR": MASK_DIR}, None),
],
"edges": [("source", "sam3", "out-1", "in-1"), ("sam3", "collect", "out-1", "in-1")],
"inputs": [
{"id": "video", "name": "左目视频", "node": "source", "port": "in-1", "items": (LEFT_VIDEOS, f"{SAMPLE_EPISODE}.mp4")},
],
},
"hand_removal_inpaint": {
"name": f"Ego 去手②ProPainter 去手({SAMPLE_EPISODE})",
"description": f"读取 {MASK_DIR} 中 SAM3.1 的掩码,对左目 {SAMPLE_EPISODE}.mp4 去手(需先运行 去手①)。",
"nodes": [
("source", "@stage", f"左目视频 {SAMPLE_EPISODE} 暂存", {"MODE": "tree", "SUBDIR": f"{STAGING_ROOT}/{LEFT_VIDEOS}"}, None),
(
"removal",
f"clone:{HAND_REMOVAL_TEMPLATE}#node-hr-1",
"人手去除",
{"INPUT_VIDEO": SAMPLE_VIDEO_PATH, "MASK_SOURCE": f"{MASK_DIR}/{SAMPLE_EPISODE}/track_result.json"},
None,
),
],
"edges": [("source", "removal", "out-1", "in-1")],
"inputs": [
{"id": "video", "name": "左目视频", "node": "source", "port": "in-1", "items": (LEFT_VIDEOS, f"{SAMPLE_EPISODE}.mp4")},
],
},
# The hand-removal image emits artifacts without ids, so the worker fails that node even
# though the cleaned video was written. This registers its outputs as proper artifacts.
"hand_removal_result": {
"name": f"Ego 去手③结果登记({SAMPLE_EPISODE})",
"description": "把 去手② 最近一次运行写在工作区的去手视频与摘要登记为产物(由平台持久化到 BOS 产物存储)。",
"nodes": [
(
"collect",
"@stage",
"去手结果登记",
{
"MODE": "collect",
"SOURCE_DIR": "@run-output:hand_removal_inpaint:node-removal",
"SUBDIR": f"/workspace/ego-staging/hand-removal/{DATASET_PREFIX}/{CAMERA_KEY}",
"EMIT_FILES": "true",
},
None,
),
],
"edges": [],
},
"full": {
"name": "Ego 全流程:校验 → 质检 → 手部分割 → 去手 → 上传",
"description": (
f"{DATASET_URI} 的完整处理链路:暂存数据集,依次做 LeRobot 概览、Parquet 结构、元数据一致性、视频帧对齐校验,"
f"再对左目 {SAMPLE_EPISODE} 做 SAM3.1 手与手臂分割;同时对 10 个左目视频做质检;"
f"两路汇合后用 ProPainter 去手,结果上传到 {OUTPUT_URI}/full/。"
),
"nodes": [
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
("summary", "operator-builtin-lerobot-summary", "LeRobot 数据集概览", {"SOURCE_PATH": STAGING_ROOT}, None),
(
"schema",
"operator-builtin-lerobot-dataset-validate",
"Parquet 结构与索引校验",
{"SOURCE_PATH": STAGING_ROOT, "REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state"},
None,
),
(
"metadata",
"operator-builtin-lerobot-metadata-consistency-validate",
"元数据一致性校验",
# Video features live in the mp4 files, not in the Parquet columns.
{"SOURCE_PATH": STAGING_ROOT, "REQUIRED_FEATURES": "action,observation.state,left_kp3d,right_kp3d"},
None,
),
(
"alignment",
"operator-builtin-lerobot-video-alignment-validate",
"视频帧对齐校验",
{"SOURCE_PATH": STAGING_ROOT, "FRAME_TOLERANCE": "1", "MAX_VIDEOS": "20"},
None,
),
("quality", "operator-builtin-video-quality", "左目视频质检", {}, None),
("sam3", f"clone:{HAND_REMOVAL_TEMPLATE}#node-sam3-1", "SAM3.1 手与手臂分割", {"VIDEO_PATH": SAMPLE_VIDEO_PATH}, None),
("collect", "@stage", "掩码汇总(等待质检与分割)", {"MODE": "collect", "SOURCE_NODE": "node-sam3", "SUBDIR": MASK_DIR}, None),
(
"removal",
f"clone:{HAND_REMOVAL_TEMPLATE}#node-hr-1",
"ProPainter 去手",
{"INPUT_VIDEO": SAMPLE_VIDEO_PATH, "MASK_SOURCE": f"{MASK_DIR}/{SAMPLE_EPISODE}/track_result.json"},
None,
),
(
"upload",
f"clone:{HAND_REMOVAL_TEMPLATE}#node-upload-1",
"去手结果上传 BOS",
{"OUTPUT_PATH": f"{OUTPUT_URI}/full/"},
"@output",
),
],
"edges": [
("stage", "summary", "out-1", "in-1"),
("summary", "schema", "out-1", "in-1"),
("schema", "metadata", "out-1", "in-1"),
("metadata", "alignment", "out-1", "in-1"),
("alignment", "sam3", "out-1", "in-1"),
("sam3", "collect", "out-1", "in-1"),
("quality", "collect", "out-1", "in-2"),
("collect", "removal", "out-1", "in-1"),
("removal", "upload", "out-1", "in-1"),
],
"inputs": [
{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")},
{"id": "videos", "name": "左目视频清单", "node": "quality", "port": "in-1", "items": (LEFT_VIDEOS, QC_GLOB)},
],
},
"cpu_data": {
"name": "Ego 数据 CPU 校验与切分",
"description": (
f"不用 GPU:把 {DATASET_URI} 暂存到工作区后,依次做 Parquet 表摘要、动作/状态数值质量、Episode 边界、"
"语言指令、Embodiment 元信息校验,并生成 train/val/test 切分清单。"
),
"nodes": [
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
(
"parquet",
"operator-builtin-parquet-table-summary",
"Parquet 表摘要",
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state", "MAX_SAMPLE_ROWS": "3"},
None,
),
(
"action_state",
"operator-builtin-action-state-quality-validate",
"动作/状态数值质量",
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "ACTION_FIELDS": "action", "STATE_FIELDS": "observation.state", "MAX_ROWS": "40000"},
None,
),
(
"boundary",
"operator-builtin-episode-boundary-validate",
"Episode 边界",
# LeRobot parquet has no terminal flag column; episodes are delimited by episode_index.
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "REQUIRE_TERMINAL_FLAG": "false", "MAX_ROWS": "40000"},
None,
),
# meta/modality.jsonl is one pretty-printed JSON object, not JSON Lines, so the
# metadata validators are pointed at specific files instead of the meta/ directory.
(
"language",
"operator-builtin-language-instruction-validate",
"语言指令",
{"SOURCE_PATH": f"{STAGING_ROOT}/meta/episodes.jsonl"},
None,
),
(
"embodiment",
"operator-builtin-embodiment-metadata-validate",
"Embodiment 元信息",
{"SOURCE_PATH": f"{STAGING_ROOT}/meta/info.json", "REQUIRED_FIELDS": "robot_type"},
None,
),
# This dataset's episodes.jsonl has one row per subtask and no `length`; roll it up first.
(
"episodes",
"@stage",
"按 episode 汇总子任务",
{"MODE": "episodes", "SOURCE_FILE": f"{STAGING_ROOT}/meta/episodes.jsonl", "SUBDIR": f"{STAGING_ROOT}/_derived/meta"},
None,
),
(
"split",
"operator-builtin-dataset-split-manifest",
"train/val/test 切分清单",
{"SOURCE_PATH": f"{STAGING_ROOT}/_derived/meta/episodes.jsonl"},
None,
),
],
"edges": [
("stage", "parquet", "out-1", "in-1"),
("parquet", "action_state", "out-1", "in-1"),
("action_state", "boundary", "out-1", "in-1"),
("boundary", "language", "out-1", "in-1"),
("language", "embodiment", "out-1", "in-1"),
("embodiment", "episodes", "out-1", "in-1"),
("episodes", "split", "out-1", "in-1"),
],
"inputs": [{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")}],
},
"cpu_video": {
"name": "Ego 视频 CPU 处理:切片 → 运动片段裁剪",
"description": f"不用 GPU:把左目视频(默认 episode_000000/000005)按 10 秒切片,再用运动检测裁出动作片段。",
"nodes": [
("slice", "operator-builtin-video-slice", "视频切片(10 秒)", {"SEGMENT_DURATION_SECONDS": "10"}, None),
(
"motion",
"operator-builtin-video-target-clip",
"运动片段裁剪",
{"DETECTION_MODE": "motion", "TARGET_LABELS": "motion", "FRAME_INTERVAL_SECONDS": "0.5", "MAX_CLIPS": "20"},
None,
),
],
"edges": [("slice", "motion", "out-1", "in-1")],
"inputs": [{"id": "videos", "name": "左目视频清单", "node": "slice", "port": "in-1", "items": (LEFT_VIDEOS, CPU_VIDEO_GLOB)}],
},
"cpu_extra": {
"name": "Ego 数据 CPU 扩展校验与格式转换",
"description": (
f"不用 GPU:直接读取已暂存的 {STAGING_ROOT},并行做 AgiBot/78 维动作/LeRobot v3/来源许可/媒体引用校验,"
"切分清单分别送入防泄漏校验与 CSV/JSON 转换,再按 split=train 路由并渲染训练集记录报告;另行校验 VLA 样本清单。"
),
"nodes": [
("agibot", "operator-builtin-agibot-world2026-validate", "AgiBot 格式(LeRobot 路径模板)校验", {"SOURCE_PATH": STAGING_ROOT}, None),
# This human_hands dataset has 78 action components, not the standard 7D RT-X schema.
(
"rtx",
"operator-builtin-rtx-action-schema-validate",
"Ego 78 维动作校验",
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "ACTION_FIELDS": "action", "EXPECTED_ACTION_DIM": "78", "REQUIRE_GRIPPER": "false"},
None,
),
("v3", "operator-builtin-lerobot-v3-episode-index-validate", "LeRobot v3 episode 索引校验", {"SOURCE_PATH": STAGING_ROOT}, None),
("provenance", "operator-builtin-dataset-provenance-audit", "来源与许可审计", {"SOURCE_PATH": STAGING_ROOT}, None),
# meta/modality.jsonl is not JSON Lines, so list the staged videos as a manifest first.
("media_list", "@stage", "生成视频引用清单", {"MODE": "media", "SOURCE_DIR": f"{STAGING_ROOT}/videos", "SUBDIR": f"{STAGING_ROOT}/_derived/media"}, None),
("media", "operator-builtin-media-reference-validate", "媒体引用校验", {"SOURCE_PATH": f"{STAGING_ROOT}/_derived/media/media.jsonl", "MAX_REFERENCES": "200"}, None),
("split", "operator-builtin-dataset-split-manifest", "train/val/test 切分清单", {"SOURCE_PATH": f"{STAGING_ROOT}/_derived/meta/episodes.jsonl"}, None),
("leakage", "operator-builtin-dataset-split-leakage-validate", "切分防泄漏校验", {}, None),
("to_csv", "operator-builtin-json-to-csv", "切分清单转 CSV", {}, None),
("to_json", "operator-builtin-csv-to-json", "CSV 回转 JSON", {}, None),
("route", "operator-builtin-conditional-route", "按 split=train 路由", {"FIELD_PATH": "split", "OPERATOR": "eq", "EXPECTED": "train"}, None),
(
"render",
"operator-builtin-template-render",
"渲染训练集报告",
{
"TEMPLATE_TEXT": "Ego train split\nFilter: {{fieldPath}} {{operator}} {{expected}}\nEpisode count: {{count}}\nEpisodes: {{items}}",
"OUTPUT_NAME": "train-split.txt",
},
None,
),
# vla-model-train only reads JSON/JSONL/CSV samples, not this dataset's Parquet, so the
# VLA chain stops at the manifest check.
("vla_samples", "operator-builtin-vla-sample-manifest-validate", "VLA 样本清单校验", {"SOURCE_PATH": f"{STAGING_ROOT}/data", "CHECK_IMAGE_EXISTS": "false", "MAX_ROWS": "2000"}, None),
],
# The platform requires one connected graph, so the independent validators fan out from
# the split node; each reads its explicit SOURCE_PATH ahead of the upstream artifact.
"edges": [
("split", "agibot", "out-1", "in-1"),
("split", "rtx", "out-1", "in-1"),
("split", "v3", "out-1", "in-1"),
("split", "provenance", "out-1", "in-1"),
("split", "media_list", "out-1", "in-1"),
("media_list", "media", "out-1", "in-1"),
("split", "vla_samples", "out-1", "in-1"),
("split", "leakage", "out-1", "in-1"),
# Convert episode records, not the leakage validator's one-row report.
("split", "to_csv", "out-1", "in-1"),
("to_csv", "to_json", "out-1", "in-1"),
("to_json", "route", "out-1", "in-1"),
("route", "render", "matched", "in-1"),
],
},
"probe_schema": {
"name": "Ego 诊断·Parquet 校验单节点",
"description": f"诊断用:直接对已暂存的 {STAGING_ROOT} 运行 Parquet 结构校验(单节点)。",
"nodes": [
(
"schema",
"operator-builtin-lerobot-dataset-validate",
"Parquet 结构与索引校验",
{
"SOURCE_PATH": STAGING_ROOT,
"REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state",
"__image": "@runner",
},
None,
),
],
"edges": [],
},
}
def presigned_files(api: Api, sub_prefix: str) -> list[dict[str, Any]]:
"""Presigned (1h) download URLs for every object under DATASET_PREFIX/sub_prefix."""
manifest = api.request(
"GET",
"/api/object-storage/directory/manifest",
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": f"{DATASET_PREFIX}/{sub_prefix}".rstrip("/")},
)
return manifest.get("files") or []
def presigned_url(api: Api, relative_key: str) -> str:
directory, _, name = relative_key.rpartition("/")
for item in presigned_files(api, directory):
if item.get("key") == f"{DATASET_PREFIX}/{relative_key}":
return item["downloadUrl"]
raise RuntimeError(f"object not found: {DATASET_URI}/{relative_key}")
def items_payload(api: Api, sub_prefix: str, glob: str) -> dict[str, Any]:
"""Inline run input: one item per object, downloadable by video operators via source.previewURL."""
items = []
for item in presigned_files(api, sub_prefix):
name = item["key"].rsplit("/", 1)[-1]
relative = item.get("relativePath") or name
if not fnmatch.fnmatch(relative, glob) or "_droboticflow/" in f"/{relative}":
continue
items.append(
{
"filename": name,
"relativePath": relative,
"sizeBytes": item.get("sizeBytes", 0),
"source": {"path": f"bos://{BUCKET}/{item['key']}", "previewURL": item["downloadUrl"]},
}
)
if not items:
raise RuntimeError(f"no objects match {DATASET_URI}/{sub_prefix}/{glob}")
return {"datasetType": "ego-video-list", "sourcePath": f"{DATASET_URI}/{sub_prefix}", "items": items}
def resolve_overrides(api: Api, overrides: dict[str, str]) -> dict[str, str]:
resolved = {}
for key, value in overrides.items():
if value == "@runner":
value = api.request("GET", "/worker-api/capacity").get("runnerImage") or ""
if not value:
raise RuntimeError("worker did not report its operator-runner image")
elif value.startswith("@url:"):
value = presigned_url(api, value[len("@url:") :])
elif value.startswith("@run-output:"):
workflow_key, _, node_id = value[len("@run-output:") :].partition(":")
run_id = load_state()["runs"].get(workflow_key)
if not run_id:
raise RuntimeError(f"run `{workflow_key}` first")
value = f"/workspace/system/runs/{run_id}/nodes/{node_id}/outputs"
resolved[key] = value
return resolved
def live_operators(api: Api) -> dict[str, dict[str, Any]]:
return {item["id"]: item for item in items_of(api.request("GET", "/api/operators"))}
def node_payload(operator: dict[str, Any], label: str, overrides: dict[str, str], pattern: str | None, index: int) -> dict[str, Any]:
hyperparams = []
seen = set()
for item in operator.get("hyperparams") or []:
entry = dict(item)
if entry.get("key") in overrides:
entry["value"] = overrides[entry["key"]]
seen.add(entry.get("key"))
hyperparams.append(entry)
for key, value in overrides.items():
if key not in seen:
hyperparams.append({"key": key, "value": value})
data: dict[str, Any] = {
"label": label,
"category": operator.get("category", ""),
"description": operator.get("description", ""),
"operatorId": operator["id"],
"operatorImageId": operator.get("imageId", ""),
"image": operator.get("image", ""),
"containerCommand": operator.get("containerCommand") or [],
"containerArgs": operator.get("containerArgs") or [],
"runtimeContract": operator.get("runtimeContract") or {},
"skillDefinition": operator.get("skillDefinition", ""),
"inputs": operator.get("inputs") or [],
"outputs": operator.get("outputs") or [],
"hyperparams": hyperparams,
}
resources = operator.get("resources") or {}
if resources.get("cpuLimit"):
data["cpu"] = resources["cpuLimit"]
if resources.get("memoryLimit"):
data["memory"] = resources["memoryLimit"]
if resources.get("gpuCount"):
data["gpu"] = resources["gpuCount"]
if pattern == "@output":
data.update({"connectionId": CONNECTION_ID, "bucket": BUCKET})
elif pattern is not None:
# Source nodes must name the storage connection; without it the worker
# treats bos:// outputs as local files (why earlier digua runs failed).
data.update({"connectionId": CONNECTION_ID, "bucket": BUCKET, "prefix": DATASET_PREFIX, "pattern": pattern})
for key, value in {
"DATA_SOURCE": "BOS",
"SOURCE_PATH": DATASET_URI,
"STORAGE_PATTERN": pattern,
"SCAN_MODE": "FULL",
}.items():
data["hyperparams"] = [h for h in data["hyperparams"] if h.get("key") != key] + [{"key": key, "value": value}]
return {"id": f"node-{label and index}-{operator['id'][-8:]}", "type": "operator", "data": data, "position": {"x": 360 * index, "y": 0}}
def cloned_operator(api: Api, reference: str, cache: dict[str, Any]) -> dict[str, Any]:
"""Turn a node of an existing workflow into an operator-shaped dict so its exact settings are reused."""
workflow_id, _, node_id = reference.removeprefix("clone:").partition("#")
if workflow_id not in cache:
cache[workflow_id] = api.request("GET", f"/api/workflows/{workflow_id}")
node = next((n for n in cache[workflow_id].get("nodes", []) if n.get("id") == node_id), None)
if node is None:
raise RuntimeError(f"template node {node_id} not found in {workflow_id}")
data = dict(node["data"])
return {**data, "id": data["operatorId"], "imageId": data.get("operatorImageId", ""), "_clone": data}
def workflow_payload(spec: dict[str, Any], operators: dict[str, dict[str, Any]], api: Api | None = None) -> dict[str, Any]:
nodes = []
ids: dict[str, str] = {}
ports: dict[str, dict[str, Any]] = {}
clone_cache: dict[str, Any] = {}
for index, (key, operator_id, label, overrides, pattern) in enumerate(spec["nodes"]):
if operator_id == "@stage":
operator_id = load_state().get("stageOperatorId") or ""
if not operator_id:
raise RuntimeError("run `register-stage` first")
if operator_id.startswith("clone:"):
if api is None:
raise RuntimeError("cloning template nodes requires an API session")
operator = cloned_operator(api, operator_id, clone_cache)
else:
operator = operators.get(operator_id)
if operator is None:
raise RuntimeError(f"operator {operator_id} is not available in this organization")
if api is not None:
overrides = resolve_overrides(api, overrides)
image = overrides.pop("__image", None) if "__image" in overrides else None
node = node_payload(operator, label, overrides, pattern, index)
if image:
node["data"]["image"] = image
node["data"]["operatorImageId"] = image
if "_clone" in operator:
# Keep every template field (cpu/memory/gpu, image, args) and only change what we override.
cloned = {**operator["_clone"], "label": label, "hyperparams": node["data"]["hyperparams"]}
for field in ("connectionId", "bucket", "prefix", "pattern"):
if field in node["data"]:
cloned[field] = node["data"][field]
node["data"] = cloned
node["id"] = f"node-{key}"
ids[key] = node["id"]
ports[key] = operator
nodes.append(node)
edges = []
for source, target, *handles in spec["edges"]:
source_handle = handles[0] if handles else (ports[source].get("outputs") or [{}])[0].get("id", "out-1")
target_handle = handles[1] if len(handles) > 1 else (ports[target].get("inputs") or [{}])[0].get("id", "in-1")
edges.append(
{
"id": f"edge-{source}-{target}-{target_handle}",
"source": ids[source],
"sourceHandle": source_handle,
"target": ids[target],
"targetHandle": target_handle,
}
)
payload = {"name": spec["name"], "description": spec["description"], "type": "batch", "nodes": nodes, "edges": edges}
if spec.get("inputs"):
payload["workflowInputs"] = [
{
"id": item["id"],
"name": item["name"],
"type": "json",
"required": True,
"bindings": [{"nodeId": ids[item["node"]], "targetPortId": item["port"]}],
}
for item in spec["inputs"]
]
return payload
def run_inputs(api: Api, spec: dict[str, Any]) -> dict[str, Any]:
return {
item["id"]: {"mode": "inline", "type": "json", "inline": items_payload(api, *item["items"])}
for item in spec.get("inputs") or []
}
def load_state() -> dict[str, Any]:
path = STATE_DIR / "state.json"
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"workflows": {}, "runs": {}}
def ensure_project(api: Api, state: dict[str, Any]) -> str:
if state.get("projectId"):
return state["projectId"]
for project in items_of(api.request("GET", "/api/projects")):
if project.get("name") == PROJECT_NAME:
state["projectId"] = project["id"]
return project["id"]
project = api.request(
"POST",
"/api/projects",
{"name": PROJECT_NAME, "description": f"第一视角(ego)数据处理:{DATASET_URI}", "status": "active"},
)
state["projectId"] = project["id"]
return project["id"]
def cmd_create(api: Api, keys: list[str], run: bool) -> None:
api.ensure_login()
resolve_connection_id(api)
state = load_state()
project_id = ensure_project(api, state)
operators = live_operators(api)
for key in keys or list(WORKFLOWS):
payload = workflow_payload(WORKFLOWS[key], operators, api)
existing = state["workflows"].get(key)
if existing:
workflow = api.request("PUT", f"/api/workflows/{existing}", payload)
print(f"updated workflow [{key}] {workflow['name']} {workflow['id']}")
else:
workflow = api.request("POST", f"/api/projects/{project_id}/workflows", payload)
state["workflows"][key] = workflow["id"]
print(f"created workflow [{key}] {workflow['name']} {workflow['id']}")
save("state.json", state)
if run:
body: dict[str, Any] = {"trigger": "manual"}
inputs = run_inputs(api, WORKFLOWS[key])
if inputs:
body["inputs"] = inputs
queued = api.request("POST", f"/api/workflows/{workflow['id']}/run", body)
state["runs"][key] = queued["id"]
save("state.json", state)
print(f" queued run {queued['id']}")
print(f"project: {PROJECT_NAME} ({project_id}) {API_BASE_URL}/projects/{project_id}")
def cmd_status(api: Api, keys: list[str], wait: bool) -> None:
api.ensure_login()
state = load_state()
targets = keys or list(state["runs"])
terminal = {"completed", "failed", "cancelled"}
while True:
report: dict[str, Any] = {}
for key in targets:
run_id = state["runs"].get(key)
if not run_id:
continue
run = api.request("GET", f"/api/runs/{run_id}")
entry = {"runId": run_id, "status": run.get("status"), "nodes": [], "raw": {k: v for k, v in run.items() if k not in ("logs", "nodeExecutions", "artifacts", "resourceSeries")}}
for node in run.get("nodeExecutions") or []:
entry["nodes"].append(
{"nodeId": node.get("nodeId"), "status": node.get("status"), "error": str(node.get("error") or "")[:1500]}
)
if run.get("status") in terminal:
try:
entry["artifacts"] = api.request("GET", f"/api/runs/{run_id}/artifacts")
except RuntimeError as error:
entry["artifacts"] = f"ERROR: {error}"
report[key] = entry
save("status.json", report)
stamp = time.strftime("%H:%M:%S")
for key, entry in report.items():
nodes = " ".join(f"{n['nodeId'].removeprefix('node-')}={n['status']}" for n in entry["nodes"])
print(f"[{stamp}] {key}: {entry['status']} {nodes}")
if not wait or all(entry["status"] in terminal for entry in report.values()):
return
time.sleep(20)
def cmd_show(api: Api) -> None:
"""Read the saved workflows back from the server; * marks nodes bound to a storage connection."""
api.ensure_login()
state = load_state()
for key, workflow_id in state["workflows"].items():
workflow = api.request("GET", f"/api/workflows/{workflow_id}")
nodes = [n["id"].removeprefix("node-") + ("*" if n["data"].get("connectionId") else "") for n in workflow["nodes"]]
print(f"[{key}] {workflow['name']} status={workflow.get('status')} edges={len(workflow['edges'])} nodes={nodes}")
def cmd_logs(api: Api, keys: list[str]) -> None:
"""Save run logs; keys are workflow keys from state.json or raw run ids."""
api.ensure_login()
state = load_state()
result = {}
for key in keys:
run_id = state["runs"].get(key, key)
result[key] = api.request("GET", f"/api/runs/{run_id}/logs")
print("saved:", save("logs.json", result))
def cmd_worker(api: Api) -> None:
"""Save worker stats/capacity to diagnose runs stuck in the queue."""
api.ensure_login()
result = {}
try:
settings = api.request("GET", "/api/settings")
result["workerSettings"] = settings.get("worker") if isinstance(settings, dict) else settings
except RuntimeError as error:
result["workerSettings"] = f"ERROR: {error}"
runs = items_of(api.request("GET", "/api/runs"))
result["activeRuns"] = [r for r in runs if r.get("status") in ("running", "queued", "stopping")]
for path in ("/worker-api/stats", "/worker-api/capacity"):
try:
result[path] = api.request("GET", path)
except RuntimeError as error:
result[path] = f"ERROR: {error}"
print("saved:", save("worker.json", result))
def cmd_artifacts(api: Api, keys: list[str]) -> None:
"""Save each run's artifacts plus the content of its JSON reports."""
api.ensure_login()
state = load_state()
result: dict[str, Any] = {}
for key in keys:
run_id = state["runs"].get(key, key)
artifacts = items_of(api.request("GET", f"/api/runs/{run_id}/artifacts"))
entries = []
for artifact in artifacts:
entry = dict(artifact)
if str(artifact.get("name", "")).endswith(".json"):
try:
entry["content"] = api.request("GET", f"/api/runs/{run_id}/artifacts/{artifact['id']}/content")
except RuntimeError as error:
entry["content"] = f"ERROR: {error}"
entries.append(entry)
result[key] = {"runId": run_id, "artifacts": entries}
print(f"{key}: {len(entries)} artifacts")
print("saved:", save("artifacts.json", result))
def cmd_stop(api: Api, keys: list[str], force: bool) -> None:
api.ensure_login()
state = load_state()
for key in keys:
run_id = state["runs"].get(key, key)
action = "force-stop" if force else "stop"
result = api.request("POST", f"/api/runs/{run_id}/{action}", {})
status = result.get("status") if isinstance(result, dict) else str(result)[:80]
print(f"{key}: {action} -> {status}")
STAGE_OPERATOR_NAME = "Ego 数据暂存(预签名下载到工作区)"
STAGE_CODE_SUBPATH = "lightweight-operators/ego-stage"
STAGE_MOUNT_PATH = "/opt/droboticflow/lightweight/ego-stage"
def ensure_stage_operator(api: Api) -> str:
"""Create (or refresh) the lightweight operator that stages presigned files onto the workspace PVC."""
state = load_state()
capacity = api.request("GET", "/worker-api/capacity")
image = capacity.get("runnerImage") or ""
if not image:
raise RuntimeError("worker did not report its operator-runner image")
payload = {
"name": STAGE_OPERATOR_NAME,
"type": "lightweight",
"category": "IO",
"description": "把工作流输入里的预签名 BOS 文件下载到当前运行的工作区目录,保持相对路径;tree 模式输出本地目录,files 模式另外逐个输出文件产物。",
"image": image,
# Not operator.py: that name would shadow the stdlib `operator` module.
"containerCommand": ["python", f"{STAGE_MOUNT_PATH}/ego_stage.py"],
"containerArgs": [],
"runtimeContract": {"lightweightOperator": {"codeSubPath": STAGE_CODE_SUBPATH, "mountPath": STAGE_MOUNT_PATH}},
# in-2 only lets a second branch join (collect mode ignores inputs).
"inputs": [{"id": "in-1", "name": "items", "type": "json"}, {"id": "in-2", "name": "join", "type": "json"}],
"outputs": [{"id": "out-1", "name": "output", "type": "json"}],
"hyperparams": [
{"key": "MODE", "value": "tree", "description": "tree=输出本地目录;files=另外逐个输出文件产物"},
{"key": "SUBDIR", "value": "dataset", "description": "输出目录下存放文件的子目录"},
],
"generationSource": "manual",
"publicationStatus": "published",
}
operator_id = state.get("stageOperatorId")
if operator_id:
api.request("PUT", f"/api/operators/{operator_id}", payload)
else:
operator_id = api.request("POST", "/api/operators", payload)["id"]
state["stageOperatorId"] = operator_id
save("state.json", state)
code = (ROOT / "lightweight-operators" / "ego-stage" / "ego_stage.py").read_text(encoding="utf-8")
api.request("PUT", f"/api/operators/{operator_id}/lightweight-files/content", {"path": "ego_stage.py", "content": code})
try:
api.request("DELETE", f"/api/operators/{operator_id}/lightweight-files", query={"path": "operator.py"})
except RuntimeError:
pass # Only present from the first registration.
return operator_id
def cmd_register_stage(api: Api) -> None:
api.ensure_login()
print("stage operator:", ensure_stage_operator(api))
def cmd_run_detail(api: Api, run_ids: list[str]) -> None:
"""Save full run records (node executions with inputs/outputs) for comparison."""
api.ensure_login()
state = load_state()
result = {key: api.request("GET", f"/api/runs/{state['runs'].get(key, key)}") for key in run_ids}
print("saved:", save("run-detail.json", result))
def cmd_download(api: Api, keys: list[str]) -> None:
"""Download objects (full keys in BUCKET) to output/ego-workflows/downloads/ via presigned URLs."""
api.ensure_login()
target_dir = STATE_DIR / "downloads"
target_dir.mkdir(parents=True, exist_ok=True)
for key in keys:
key = key.removeprefix(f"bos://{BUCKET}/")
directory = key.rsplit("/", 1)[0]
manifest = api.request(
"GET",
"/api/object-storage/directory/manifest",
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": directory},
)
match = next((item for item in manifest.get("files") or [] if item.get("key") == key), None)
if match is None:
raise RuntimeError(f"object not found: {key}")
target = target_dir / key.rsplit("/", 1)[-1]
with urllib.request.urlopen(match["downloadUrl"], timeout=600) as response, target.open("wb") as handle:
handle.write(response.read())
print(f"downloaded {target} ({target.stat().st_size} bytes)")
def cmd_pin_operator(api: Api, refs: list[str]) -> None:
"""Repin builtin operators to image@sha256 refs (operator id derived from the repository name)."""
api.ensure_login()
for ref in refs:
repository, _, digest = ref.partition("@")
if not digest.startswith("sha256:"):
raise RuntimeError(f"expected image@sha256:..., got {ref}")
operator_id = "operator-builtin-" + repository.rsplit("/", 1)[-1].removeprefix("droboticflow-operator-")
before = api.request("GET", "/api/operators").get("items", [])
previous = next((item.get("imageId") for item in before if item.get("id") == operator_id), None)
updated = api.request("PUT", f"/api/operators/{operator_id}", {"image": f"{repository}:builtin", "imageId": ref})
print(f"{operator_id}: {previous} -> {updated.get('imageId')}")
def cmd_probe(api: Api) -> None:
api.ensure_login()
connections = api.request("GET", "/api/storage/connections")
result: dict[str, Any] = {"connections": connections, "byConnection": {}}
items = connections.get("items", connections) if isinstance(connections, dict) else connections
for connection in items or []:
connection_id = connection.get("id", "")
entry: dict[str, Any] = {}
for label, path, query in [
("overview", "/api/object-storage", {"connectionId": connection_id}),
("root", "/api/object-storage/objects", {"connectionId": connection_id, "bucket": BUCKET, "prefix": "", "limit": 200}),
("dataset", "/api/object-storage/objects", {"connectionId": connection_id, "bucket": BUCKET, "prefix": DATASET_PREFIX + "/", "limit": 200}),
]:
try:
entry[label] = api.request("GET", path, query=query)
except RuntimeError as error:
entry[label] = f"ERROR: {error}"
result["byConnection"][connection_id] = entry
print("connections:", len(items or []))
print("saved:", save("probe.json", result))
def cmd_cat(api: Api, keys: list[str]) -> None:
api.ensure_login()
result = {}
for key in keys:
full = key if key.startswith(DATASET_PREFIX + "/") else f"{DATASET_PREFIX}/{key}"
result[full] = read_text_object(api, full)
print("saved:", save("cat.json", result))
def items_of(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, dict):
return payload.get("items") or []
return payload or []
def cmd_existing(api: Api) -> None:
"""Dump the org's workflows that touch the dataset, plus their latest runs."""
api.ensure_login()
matches = []
projects = items_of(api.request("GET", "/api/projects"))
for project in projects:
for summary in items_of(api.request("GET", f"/api/projects/{project['id']}/workflows")):
workflow = api.request("GET", f"/api/workflows/{summary['id']}")
text = json.dumps(workflow, ensure_ascii=False)
if "digua_test" not in text and "ego" not in text.lower():
continue
runs = items_of(api.request("GET", f"/api/workflows/{summary['id']}/runs"))[:3]
matches.append({"project": project.get("name"), "workflow": workflow, "runs": runs})
print(f"projects: {len(projects)}, matching workflows: {len(matches)}")
for match in matches:
statuses = [run.get("status") for run in match["runs"]]
print(f" [{match['project']}] {match['workflow'].get('name')} runs={statuses}")
print("saved:", save("existing.json", {"projects": projects, "matches": matches}))
def cmd_details(api: Api, run_ids: list[str]) -> None:
"""Save run details (status, node errors) plus the live operator catalog."""
api.ensure_login()
result: dict[str, Any] = {"runs": {}, "operators": api.request("GET", "/api/operators")}
for run_id in run_ids:
try:
result["runs"][run_id] = api.request("GET", f"/api/runs/{run_id}")
except RuntimeError as error:
result["runs"][run_id] = f"ERROR: {error}"
print("operators:", len(items_of(result["operators"])), "runs:", len(result["runs"]))
print("saved:", save("details.json", result))
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("command", choices=["inspect", "probe", "cat", "existing", "details", "create", "status", "show", "logs", "worker", "artifacts", "stop", "register-stage", "run-detail", "download", "pin-operator"])
parser.add_argument("keys", nargs="*", help="object keys (cat), run ids (details) or workflow keys (create/status)")
parser.add_argument("--run", action="store_true", help="create: start a run after saving each workflow")
parser.add_argument("--wait", action="store_true", help="status: poll until every run finishes")
parser.add_argument("--force", action="store_true", help="stop: force-stop instead of a graceful stop")
args = parser.parse_args()
api = Api()
if args.command == "inspect":
cmd_inspect(api)
elif args.command == "probe":
cmd_probe(api)
elif args.command == "existing":
cmd_existing(api)
elif args.command == "create":
cmd_create(api, args.keys, args.run)
elif args.command == "pin-operator":
cmd_pin_operator(api, args.keys)
elif args.command == "download":
cmd_download(api, args.keys)
elif args.command == "run-detail":
cmd_run_detail(api, args.keys)
elif args.command == "register-stage":
cmd_register_stage(api)
elif args.command == "stop":
cmd_stop(api, args.keys, args.force)
elif args.command == "artifacts":
cmd_artifacts(api, args.keys)
elif args.command == "worker":
cmd_worker(api)
elif args.command == "logs":
cmd_logs(api, args.keys)
elif args.command == "show":
cmd_show(api)
elif args.command == "status":
cmd_status(api, args.keys, args.wait)
elif args.command == "details":
cmd_details(api, args.keys)
elif args.command == "cat":
cmd_cat(api, args.keys)
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except RuntimeError as error:
print("ERROR:", error, file=sys.stderr)
sys.exit(1)