Validate the dataset's 78-dimensional actions, export split records instead of the leakage report, and render selected training episodes with supported template fields. Add regression coverage for record round-tripping, route counts, and empty reports. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1161 lines
55 KiB
Python
1161 lines
55 KiB
Python
#!/usr/bin/env python3
|
||
"""Inspect an egocentric LeRobot dataset on BOS and build processing workflows for it.
|
||
|
||
Usage:
|
||
python3 scripts/ego_workflows.py inspect
|
||
python3 scripts/ego_workflows.py create [KEY ...] [--run]
|
||
python3 scripts/ego_workflows.py status [KEY ...] [--wait]
|
||
|
||
Login: DROBOTICFLOW_EMAIL / DROBOTICFLOW_PASSWORD, otherwise prompted. The
|
||
session cookie is kept in output/ego-workflows/ so later steps reuse it.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import fnmatch
|
||
import getpass
|
||
import http.client
|
||
import http.cookiejar
|
||
import json
|
||
import os
|
||
import pathlib
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from typing import Any
|
||
|
||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||
STATE_DIR = ROOT / "output" / "ego-workflows"
|
||
COOKIE_FILE = STATE_DIR / "cookies.txt"
|
||
API_BASE_URL = os.environ.get("API_BASE_URL", "http://120.48.164.19").rstrip("/")
|
||
|
||
BUCKET = os.environ.get("EGO_BUCKET", "testxyz")
|
||
DATASET_PREFIX = os.environ.get("EGO_DATASET_PREFIX", "digua_test/R16to9_10_sample").strip("/")
|
||
CAMERA_KEY = os.environ.get("EGO_CAMERA_KEY", "observation.images.cam_head_ego_left")
|
||
PROJECT_NAME = os.environ.get("EGO_PROJECT_NAME", "Ego 数据处理")
|
||
CONNECTION_ID = os.environ.get("EGO_CONNECTION_ID", "")
|
||
|
||
|
||
class Api:
|
||
def __init__(self) -> None:
|
||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||
self.jar = http.cookiejar.MozillaCookieJar(str(COOKIE_FILE))
|
||
if COOKIE_FILE.exists():
|
||
self.jar.load(ignore_discard=True, ignore_expires=True)
|
||
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.jar))
|
||
|
||
def _csrf(self) -> str:
|
||
for cookie in self.jar:
|
||
if cookie.name == "droboticflow_csrf":
|
||
return cookie.value or ""
|
||
return ""
|
||
|
||
def request(self, method: str, path: str, body: Any = None, query: dict[str, Any] | None = None) -> Any:
|
||
url = API_BASE_URL + path
|
||
if query:
|
||
url += "?" + urllib.parse.urlencode({k: v for k, v in query.items() if v not in (None, "")})
|
||
data = None
|
||
headers = {"Accept": "application/json"}
|
||
if body is not None:
|
||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||
headers["Content-Type"] = "application/json"
|
||
if method not in {"GET", "HEAD"} and self._csrf():
|
||
headers["X-Droboticflow-CSRF-Token"] = self._csrf()
|
||
request = urllib.request.Request(url, data=data, method=method, headers=headers)
|
||
try:
|
||
with self.opener.open(request, timeout=120) as response:
|
||
try:
|
||
raw = response.read()
|
||
except http.client.IncompleteRead as error:
|
||
raw = error.partial # the stop endpoints sometimes cut the chunked body short
|
||
except urllib.error.HTTPError as error:
|
||
detail = error.read().decode("utf-8", "replace")[:800]
|
||
raise RuntimeError(f"{method} {path} -> HTTP {error.code}: {detail}") from None
|
||
finally:
|
||
self.jar.save(ignore_discard=True, ignore_expires=True)
|
||
if not raw:
|
||
return None
|
||
try:
|
||
return json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return raw.decode("utf-8", "replace")
|
||
|
||
def ensure_login(self) -> dict[str, Any]:
|
||
try:
|
||
me = self.request("GET", "/api/auth/me")
|
||
if isinstance(me, dict) and (me.get("user") or me.get("id")):
|
||
return me
|
||
except RuntimeError:
|
||
pass
|
||
email = os.environ.get("DROBOTICFLOW_EMAIL") or input("Droboticflow 登录邮箱: ").strip()
|
||
password = os.environ.get("DROBOTICFLOW_PASSWORD") or getpass.getpass("密码: ")
|
||
self.request("POST", "/api/auth/login", {"email": email, "password": password})
|
||
return self.request("GET", "/api/auth/me")
|
||
|
||
|
||
def save(name: str, payload: Any) -> pathlib.Path:
|
||
path = STATE_DIR / name
|
||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def resolve_connection_id(api: Api) -> str:
|
||
"""Pick the storage connection that exposes BUCKET (the org default may be unconfigured)."""
|
||
global CONNECTION_ID
|
||
if CONNECTION_ID:
|
||
return CONNECTION_ID
|
||
connections = api.request("GET", "/api/storage/connections")
|
||
items = connections.get("items", connections) if isinstance(connections, dict) else connections
|
||
for connection in items or []:
|
||
if connection.get("bucket") == BUCKET:
|
||
CONNECTION_ID = connection["id"]
|
||
return CONNECTION_ID
|
||
for connection in items or []:
|
||
overview = api.request("GET", "/api/object-storage", query={"connectionId": connection["id"]})
|
||
if BUCKET in [bucket.get("name") for bucket in overview.get("buckets") or []]:
|
||
CONNECTION_ID = connection["id"]
|
||
return CONNECTION_ID
|
||
raise RuntimeError(f"no storage connection exposes bucket {BUCKET}")
|
||
|
||
|
||
def list_directory(api: Api, prefix: str) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
marker = ""
|
||
while True:
|
||
page = api.request(
|
||
"GET",
|
||
"/api/object-storage/objects",
|
||
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": prefix, "marker": marker, "limit": 1000},
|
||
)
|
||
items.extend(page.get("items") or [])
|
||
marker = page.get("nextMarker") or ""
|
||
if not page.get("hasMore") or not marker:
|
||
return items
|
||
|
||
|
||
def list_objects(api: Api, prefix: str, limit: int = 20000) -> list[dict[str, Any]]:
|
||
"""Walk the directory-style listing into a flat list of files."""
|
||
files: list[dict[str, Any]] = []
|
||
pending = [prefix.rstrip("/") + "/"]
|
||
while pending and len(files) < limit:
|
||
current = pending.pop(0)
|
||
for item in list_directory(api, current):
|
||
if item.get("kind") == "directory":
|
||
pending.append(str(item["key"]).rstrip("/") + "/")
|
||
else:
|
||
files.append(item)
|
||
return files
|
||
|
||
|
||
def read_text_object(api: Api, key: str) -> str:
|
||
content = api.request(
|
||
"GET", "/api/object-storage/content", query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "key": key}
|
||
)
|
||
return content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
|
||
|
||
|
||
def cmd_inspect(api: Api) -> None:
|
||
me = api.ensure_login()
|
||
print("logged in:", json.dumps(me, ensure_ascii=False)[:200])
|
||
objects = list_objects(api, DATASET_PREFIX + "/")
|
||
keys = [str(item.get("key") or item.get("name") or "") for item in objects]
|
||
print(f"objects under bos://{BUCKET}/{DATASET_PREFIX}/: {len(keys)}")
|
||
meta: dict[str, Any] = {}
|
||
for key in keys:
|
||
if key.endswith((".json", ".jsonl")) and "/meta/" in f"/{key}" and len(meta) < 12:
|
||
try:
|
||
meta[key] = read_text_object(api, key)[:20000]
|
||
except RuntimeError as error:
|
||
meta[key] = f"ERROR: {error}"
|
||
path = save(
|
||
"inspect.json",
|
||
{"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": DATASET_PREFIX, "objects": objects, "meta": meta},
|
||
)
|
||
dirs: dict[str, int] = {}
|
||
for key in keys:
|
||
rel = key[len(DATASET_PREFIX) + 1 :] if key.startswith(DATASET_PREFIX + "/") else key
|
||
top = "/".join(rel.split("/")[:-1]) or "."
|
||
dirs[top] = dirs.get(top, 0) + 1
|
||
for name, count in sorted(dirs.items()):
|
||
print(f" {name}/ ({count} files)")
|
||
print("saved:", path)
|
||
|
||
# --- workflow definitions -------------------------------------------------
|
||
|
||
DATASET_URI = f"bos://{BUCKET}/{DATASET_PREFIX}"
|
||
OUTPUT_URI = os.environ.get("EGO_OUTPUT_URI", f"bos://{BUCKET}/{DATASET_PREFIX}_processed").rstrip("/")
|
||
LEFT_VIDEOS = f"videos/chunk-000/{CAMERA_KEY}"
|
||
SAMPLE_EPISODE = os.environ.get("EGO_SAMPLE_EPISODE", "episode_000000")
|
||
QC_GLOB = os.environ.get("EGO_QC_GLOB", "*.mp4")
|
||
CPU_VIDEO_GLOB = os.environ.get("EGO_CPU_VIDEO_GLOB", "episode_00000[05].mp4")
|
||
# Proven egodex pipeline (SAM3.1 v6 -> hand removal v4 -> upload) whose node settings are cloned.
|
||
# Validator images from completed acceptance runs; the live catalog's `:builtin` digests never start.
|
||
SCHEMA_TEMPLATE = "workflow-23533841935b7e283183138b19825b0f#op-lerobot-dataset-validate"
|
||
METADATA_TEMPLATE = "workflow-602cefb887ee6c1f189c89e395f82b44#op-lerobot-metadata-consistency-validate"
|
||
ALIGNMENT_TEMPLATE = "workflow-602cefb887ee6c1f189c89e395f82b44#op-lerobot-video-alignment-validate"
|
||
# Stable staging dir on the workspace PVC for operators that only read local paths (SAM3, hand removal).
|
||
STAGING_ROOT = f"/workspace/ego-staging/{DATASET_PREFIX}"
|
||
SAMPLE_VIDEO_PATH = f"{STAGING_ROOT}/{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4"
|
||
MASK_DIR = f"/workspace/ego-staging/masks/{DATASET_PREFIX}/{CAMERA_KEY}"
|
||
HAND_REMOVAL_TEMPLATE = os.environ.get("EGO_HAND_REMOVAL_TEMPLATE", "workflow-a77bd54acf37717a2a72339021182072")
|
||
|
||
# Each node: (node id, operator id, label, hyperparam overrides, BOS source pattern or None).
|
||
# Edges connect each operator's first output to the next operator's first input.
|
||
WORKFLOWS: dict[str, dict[str, Any]] = {
|
||
"smoke": {
|
||
"name": "Ego 冒烟-元数据读取",
|
||
"description": f"读取 {DATASET_URI} 的 meta/ 与 data/(不含视频),生成 LeRobot 数据集概览,用于验证 BOS 数据接入链路。",
|
||
"nodes": [
|
||
("source", "operator-builtin-real-source", "Ego 元数据与 Parquet", {}, "[md][ea]ta/*"),
|
||
("summary", "operator-builtin-lerobot-summary", "LeRobot 数据集概览", {}, None),
|
||
],
|
||
"edges": [("source", "summary")],
|
||
},
|
||
# Each check is its own two-node workflow (stage -> one validator). On the current POC
|
||
# worker, a third node in a chain or parallel branches stall before their Kubernetes Job
|
||
# is created; stage -> one consumer runs reliably. Staged files are reused across runs.
|
||
**{
|
||
key: {
|
||
"name": f"Ego 数据集校验·{title}",
|
||
"description": f"把 {DATASET_URI} 的 meta/、data/、videos/ 暂存到工作区 {STAGING_ROOT}(已暂存的文件复用),再执行{title}。",
|
||
"nodes": [
|
||
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
|
||
(node, operator, title, {"SOURCE_PATH": STAGING_ROOT, **overrides}, None),
|
||
],
|
||
"edges": [("stage", node, "out-1", "in-1")],
|
||
"inputs": [{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")}],
|
||
}
|
||
for key, node, title, operator, overrides in [
|
||
("integrity", "summary", "LeRobot 数据集概览", "operator-builtin-lerobot-summary", {}),
|
||
(
|
||
"integrity_schema",
|
||
"schema",
|
||
"Parquet 结构与索引校验",
|
||
"operator-builtin-lerobot-dataset-validate",
|
||
{"REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state"},
|
||
),
|
||
(
|
||
"integrity_metadata",
|
||
"metadata",
|
||
"元数据一致性校验",
|
||
"operator-builtin-lerobot-metadata-consistency-validate",
|
||
{"REQUIRED_FEATURES": f"action,observation.state,left_kp3d,right_kp3d,{CAMERA_KEY}"},
|
||
),
|
||
(
|
||
"integrity_alignment",
|
||
"alignment",
|
||
"视频帧对齐校验",
|
||
"operator-builtin-lerobot-video-alignment-validate",
|
||
{"FRAME_TOLERANCE": "1", "MAX_VIDEOS": "20"},
|
||
),
|
||
]
|
||
},
|
||
"left_qc": {
|
||
"name": "Ego 左目视频质检",
|
||
"description": (
|
||
f"对 {LEFT_VIDEOS} 的全部视频计算清晰度、抖动与无手率,并按阈值判定通过/不通过。"
|
||
"视频清单在每次运行时作为工作流输入传入(含 1 小时有效的 BOS 预签名下载地址)。"
|
||
),
|
||
# No separate filter node: video-quality already marks each item passed/failed, and on
|
||
# the current worker a node consuming its externalized dataRef output never launches.
|
||
"nodes": [
|
||
("quality", "operator-builtin-video-quality", "视频质检(含通过/不通过判定)", {}, None),
|
||
],
|
||
"edges": [],
|
||
"inputs": [
|
||
{"id": "videos", "name": "左目视频清单", "node": "quality", "port": "in-1", "items": (LEFT_VIDEOS, QC_GLOB)},
|
||
],
|
||
},
|
||
"wilor": {
|
||
"name": f"Ego 手部 3D 姿态估计(WiLoR,{SAMPLE_EPISODE})",
|
||
"description": (
|
||
f"对左目 {SAMPLE_EPISODE}.mp4 抽帧调用 WiLoR 估计双手 3D 姿态,结果上传到 {OUTPUT_URI}/wilor/。"
|
||
"INPUT_URL 在每次运行前刷新为 1 小时有效的 BOS 预签名地址。"
|
||
),
|
||
"nodes": [
|
||
(
|
||
"source",
|
||
"operator-builtin-real-source",
|
||
"Ego 视频引用",
|
||
{"DATA_SOURCE": "INLINE", "SOURCE_JSON": json.dumps({"video": f"{DATASET_URI}/{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4"})},
|
||
None,
|
||
),
|
||
(
|
||
"wilor",
|
||
"operator-builtin-wilor-hand-pose",
|
||
"WiLoR 3D 手部姿态",
|
||
{
|
||
"INPUT_URL": f"@url:{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4",
|
||
"INPUT_PATH": "",
|
||
"MEDIA_KIND": "video",
|
||
"FRAME_STRIDE": "3",
|
||
"MAX_FRAMES": "120",
|
||
},
|
||
None,
|
||
),
|
||
("upload", "operator-builtin-upload", "结果上传 BOS", {"OUTPUT_PATH": f"{OUTPUT_URI}/wilor/"}, "@output"),
|
||
],
|
||
"edges": [("source", "wilor"), ("wilor", "upload")],
|
||
},
|
||
# Hand removal is split in two for the same worker limitation. SAM3 and the hand-removal
|
||
# image only read local paths (artifacts arrive as bos:// refs), so the video and the masks
|
||
# go through stable workspace paths. Results are persisted by the worker's artifact storage.
|
||
"hand_removal": {
|
||
"name": f"Ego 去手①SAM3.1 手与手臂分割({SAMPLE_EPISODE})",
|
||
"description": f"把左目 {SAMPLE_EPISODE}.mp4 暂存到 {STAGING_ROOT},用 SAM3.1 分割手与手臂,掩码汇总到 {MASK_DIR}。",
|
||
"nodes": [
|
||
("source", "@stage", f"左目视频 {SAMPLE_EPISODE} 暂存", {"MODE": "tree", "SUBDIR": f"{STAGING_ROOT}/{LEFT_VIDEOS}"}, None),
|
||
("sam3", f"clone:{HAND_REMOVAL_TEMPLATE}#node-sam3-1", "SAM3.1 手与手臂分割", {"VIDEO_PATH": SAMPLE_VIDEO_PATH}, None),
|
||
("collect", "@stage", "SAM3 掩码汇总到工作区", {"MODE": "collect", "SOURCE_NODE": "node-sam3", "SUBDIR": MASK_DIR}, None),
|
||
],
|
||
"edges": [("source", "sam3", "out-1", "in-1"), ("sam3", "collect", "out-1", "in-1")],
|
||
"inputs": [
|
||
{"id": "video", "name": "左目视频", "node": "source", "port": "in-1", "items": (LEFT_VIDEOS, f"{SAMPLE_EPISODE}.mp4")},
|
||
],
|
||
},
|
||
"hand_removal_inpaint": {
|
||
"name": f"Ego 去手②ProPainter 去手({SAMPLE_EPISODE})",
|
||
"description": f"读取 {MASK_DIR} 中 SAM3.1 的掩码,对左目 {SAMPLE_EPISODE}.mp4 去手(需先运行 去手①)。",
|
||
"nodes": [
|
||
("source", "@stage", f"左目视频 {SAMPLE_EPISODE} 暂存", {"MODE": "tree", "SUBDIR": f"{STAGING_ROOT}/{LEFT_VIDEOS}"}, None),
|
||
(
|
||
"removal",
|
||
f"clone:{HAND_REMOVAL_TEMPLATE}#node-hr-1",
|
||
"人手去除",
|
||
{"INPUT_VIDEO": SAMPLE_VIDEO_PATH, "MASK_SOURCE": f"{MASK_DIR}/{SAMPLE_EPISODE}/track_result.json"},
|
||
None,
|
||
),
|
||
],
|
||
"edges": [("source", "removal", "out-1", "in-1")],
|
||
"inputs": [
|
||
{"id": "video", "name": "左目视频", "node": "source", "port": "in-1", "items": (LEFT_VIDEOS, f"{SAMPLE_EPISODE}.mp4")},
|
||
],
|
||
},
|
||
# The hand-removal image emits artifacts without ids, so the worker fails that node even
|
||
# though the cleaned video was written. This registers its outputs as proper artifacts.
|
||
"hand_removal_result": {
|
||
"name": f"Ego 去手③结果登记({SAMPLE_EPISODE})",
|
||
"description": "把 去手② 最近一次运行写在工作区的去手视频与摘要登记为产物(由平台持久化到 BOS 产物存储)。",
|
||
"nodes": [
|
||
(
|
||
"collect",
|
||
"@stage",
|
||
"去手结果登记",
|
||
{
|
||
"MODE": "collect",
|
||
"SOURCE_DIR": "@run-output:hand_removal_inpaint:node-removal",
|
||
"SUBDIR": f"/workspace/ego-staging/hand-removal/{DATASET_PREFIX}/{CAMERA_KEY}",
|
||
"EMIT_FILES": "true",
|
||
},
|
||
None,
|
||
),
|
||
],
|
||
"edges": [],
|
||
},
|
||
"full": {
|
||
"name": "Ego 全流程:校验 → 质检 → 手部分割 → 去手 → 上传",
|
||
"description": (
|
||
f"{DATASET_URI} 的完整处理链路:暂存数据集,依次做 LeRobot 概览、Parquet 结构、元数据一致性、视频帧对齐校验,"
|
||
f"再对左目 {SAMPLE_EPISODE} 做 SAM3.1 手与手臂分割;同时对 10 个左目视频做质检;"
|
||
f"两路汇合后用 ProPainter 去手,结果上传到 {OUTPUT_URI}/full/。"
|
||
),
|
||
"nodes": [
|
||
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
|
||
("summary", "operator-builtin-lerobot-summary", "LeRobot 数据集概览", {"SOURCE_PATH": STAGING_ROOT}, None),
|
||
(
|
||
"schema",
|
||
"operator-builtin-lerobot-dataset-validate",
|
||
"Parquet 结构与索引校验",
|
||
{"SOURCE_PATH": STAGING_ROOT, "REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state"},
|
||
None,
|
||
),
|
||
(
|
||
"metadata",
|
||
"operator-builtin-lerobot-metadata-consistency-validate",
|
||
"元数据一致性校验",
|
||
# Video features live in the mp4 files, not in the Parquet columns.
|
||
{"SOURCE_PATH": STAGING_ROOT, "REQUIRED_FEATURES": "action,observation.state,left_kp3d,right_kp3d"},
|
||
None,
|
||
),
|
||
(
|
||
"alignment",
|
||
"operator-builtin-lerobot-video-alignment-validate",
|
||
"视频帧对齐校验",
|
||
{"SOURCE_PATH": STAGING_ROOT, "FRAME_TOLERANCE": "1", "MAX_VIDEOS": "20"},
|
||
None,
|
||
),
|
||
("quality", "operator-builtin-video-quality", "左目视频质检", {}, None),
|
||
("sam3", f"clone:{HAND_REMOVAL_TEMPLATE}#node-sam3-1", "SAM3.1 手与手臂分割", {"VIDEO_PATH": SAMPLE_VIDEO_PATH}, None),
|
||
("collect", "@stage", "掩码汇总(等待质检与分割)", {"MODE": "collect", "SOURCE_NODE": "node-sam3", "SUBDIR": MASK_DIR}, None),
|
||
(
|
||
"removal",
|
||
f"clone:{HAND_REMOVAL_TEMPLATE}#node-hr-1",
|
||
"ProPainter 去手",
|
||
{"INPUT_VIDEO": SAMPLE_VIDEO_PATH, "MASK_SOURCE": f"{MASK_DIR}/{SAMPLE_EPISODE}/track_result.json"},
|
||
None,
|
||
),
|
||
(
|
||
"upload",
|
||
f"clone:{HAND_REMOVAL_TEMPLATE}#node-upload-1",
|
||
"去手结果上传 BOS",
|
||
{"OUTPUT_PATH": f"{OUTPUT_URI}/full/"},
|
||
"@output",
|
||
),
|
||
],
|
||
"edges": [
|
||
("stage", "summary", "out-1", "in-1"),
|
||
("summary", "schema", "out-1", "in-1"),
|
||
("schema", "metadata", "out-1", "in-1"),
|
||
("metadata", "alignment", "out-1", "in-1"),
|
||
("alignment", "sam3", "out-1", "in-1"),
|
||
("sam3", "collect", "out-1", "in-1"),
|
||
("quality", "collect", "out-1", "in-2"),
|
||
("collect", "removal", "out-1", "in-1"),
|
||
("removal", "upload", "out-1", "in-1"),
|
||
],
|
||
"inputs": [
|
||
{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")},
|
||
{"id": "videos", "name": "左目视频清单", "node": "quality", "port": "in-1", "items": (LEFT_VIDEOS, QC_GLOB)},
|
||
],
|
||
},
|
||
"cpu_data": {
|
||
"name": "Ego 数据 CPU 校验与切分",
|
||
"description": (
|
||
f"不用 GPU:把 {DATASET_URI} 暂存到工作区后,依次做 Parquet 表摘要、动作/状态数值质量、Episode 边界、"
|
||
"语言指令、Embodiment 元信息校验,并生成 train/val/test 切分清单。"
|
||
),
|
||
"nodes": [
|
||
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
|
||
(
|
||
"parquet",
|
||
"operator-builtin-parquet-table-summary",
|
||
"Parquet 表摘要",
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state", "MAX_SAMPLE_ROWS": "3"},
|
||
None,
|
||
),
|
||
(
|
||
"action_state",
|
||
"operator-builtin-action-state-quality-validate",
|
||
"动作/状态数值质量",
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "ACTION_FIELDS": "action", "STATE_FIELDS": "observation.state", "MAX_ROWS": "40000"},
|
||
None,
|
||
),
|
||
(
|
||
"boundary",
|
||
"operator-builtin-episode-boundary-validate",
|
||
"Episode 边界",
|
||
# LeRobot parquet has no terminal flag column; episodes are delimited by episode_index.
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "REQUIRE_TERMINAL_FLAG": "false", "MAX_ROWS": "40000"},
|
||
None,
|
||
),
|
||
# meta/modality.jsonl is one pretty-printed JSON object, not JSON Lines, so the
|
||
# metadata validators are pointed at specific files instead of the meta/ directory.
|
||
(
|
||
"language",
|
||
"operator-builtin-language-instruction-validate",
|
||
"语言指令",
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/meta/episodes.jsonl"},
|
||
None,
|
||
),
|
||
(
|
||
"embodiment",
|
||
"operator-builtin-embodiment-metadata-validate",
|
||
"Embodiment 元信息",
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/meta/info.json", "REQUIRED_FIELDS": "robot_type"},
|
||
None,
|
||
),
|
||
# This dataset's episodes.jsonl has one row per subtask and no `length`; roll it up first.
|
||
(
|
||
"episodes",
|
||
"@stage",
|
||
"按 episode 汇总子任务",
|
||
{"MODE": "episodes", "SOURCE_FILE": f"{STAGING_ROOT}/meta/episodes.jsonl", "SUBDIR": f"{STAGING_ROOT}/_derived/meta"},
|
||
None,
|
||
),
|
||
(
|
||
"split",
|
||
"operator-builtin-dataset-split-manifest",
|
||
"train/val/test 切分清单",
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/_derived/meta/episodes.jsonl"},
|
||
None,
|
||
),
|
||
],
|
||
"edges": [
|
||
("stage", "parquet", "out-1", "in-1"),
|
||
("parquet", "action_state", "out-1", "in-1"),
|
||
("action_state", "boundary", "out-1", "in-1"),
|
||
("boundary", "language", "out-1", "in-1"),
|
||
("language", "embodiment", "out-1", "in-1"),
|
||
("embodiment", "episodes", "out-1", "in-1"),
|
||
("episodes", "split", "out-1", "in-1"),
|
||
],
|
||
"inputs": [{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")}],
|
||
},
|
||
"cpu_video": {
|
||
"name": "Ego 视频 CPU 处理:切片 → 运动片段裁剪",
|
||
"description": f"不用 GPU:把左目视频(默认 episode_000000/000005)按 10 秒切片,再用运动检测裁出动作片段。",
|
||
"nodes": [
|
||
("slice", "operator-builtin-video-slice", "视频切片(10 秒)", {"SEGMENT_DURATION_SECONDS": "10"}, None),
|
||
(
|
||
"motion",
|
||
"operator-builtin-video-target-clip",
|
||
"运动片段裁剪",
|
||
{"DETECTION_MODE": "motion", "TARGET_LABELS": "motion", "FRAME_INTERVAL_SECONDS": "0.5", "MAX_CLIPS": "20"},
|
||
None,
|
||
),
|
||
],
|
||
"edges": [("slice", "motion", "out-1", "in-1")],
|
||
"inputs": [{"id": "videos", "name": "左目视频清单", "node": "slice", "port": "in-1", "items": (LEFT_VIDEOS, CPU_VIDEO_GLOB)}],
|
||
},
|
||
"cpu_extra": {
|
||
"name": "Ego 数据 CPU 扩展校验与格式转换",
|
||
"description": (
|
||
f"不用 GPU:直接读取已暂存的 {STAGING_ROOT},并行做 AgiBot/78 维动作/LeRobot v3/来源许可/媒体引用校验,"
|
||
"切分清单分别送入防泄漏校验与 CSV/JSON 转换,再按 split=train 路由并渲染训练集记录报告;另行校验 VLA 样本清单。"
|
||
),
|
||
"nodes": [
|
||
("agibot", "operator-builtin-agibot-world2026-validate", "AgiBot 格式(LeRobot 路径模板)校验", {"SOURCE_PATH": STAGING_ROOT}, None),
|
||
# This human_hands dataset has 78 action components, not the standard 7D RT-X schema.
|
||
(
|
||
"rtx",
|
||
"operator-builtin-rtx-action-schema-validate",
|
||
"Ego 78 维动作校验",
|
||
{"SOURCE_PATH": f"{STAGING_ROOT}/data", "ACTION_FIELDS": "action", "EXPECTED_ACTION_DIM": "78", "REQUIRE_GRIPPER": "false"},
|
||
None,
|
||
),
|
||
("v3", "operator-builtin-lerobot-v3-episode-index-validate", "LeRobot v3 episode 索引校验", {"SOURCE_PATH": STAGING_ROOT}, None),
|
||
("provenance", "operator-builtin-dataset-provenance-audit", "来源与许可审计", {"SOURCE_PATH": STAGING_ROOT}, None),
|
||
# meta/modality.jsonl is not JSON Lines, so list the staged videos as a manifest first.
|
||
("media_list", "@stage", "生成视频引用清单", {"MODE": "media", "SOURCE_DIR": f"{STAGING_ROOT}/videos", "SUBDIR": f"{STAGING_ROOT}/_derived/media"}, None),
|
||
("media", "operator-builtin-media-reference-validate", "媒体引用校验", {"SOURCE_PATH": f"{STAGING_ROOT}/_derived/media/media.jsonl", "MAX_REFERENCES": "200"}, None),
|
||
("split", "operator-builtin-dataset-split-manifest", "train/val/test 切分清单", {"SOURCE_PATH": f"{STAGING_ROOT}/_derived/meta/episodes.jsonl"}, None),
|
||
("leakage", "operator-builtin-dataset-split-leakage-validate", "切分防泄漏校验", {}, None),
|
||
("to_csv", "operator-builtin-json-to-csv", "切分清单转 CSV", {}, None),
|
||
("to_json", "operator-builtin-csv-to-json", "CSV 回转 JSON", {}, None),
|
||
("route", "operator-builtin-conditional-route", "按 split=train 路由", {"FIELD_PATH": "split", "OPERATOR": "eq", "EXPECTED": "train"}, None),
|
||
(
|
||
"render",
|
||
"operator-builtin-template-render",
|
||
"渲染训练集报告",
|
||
{
|
||
"TEMPLATE_TEXT": "Ego train split\nFilter: {{fieldPath}} {{operator}} {{expected}}\nEpisode count: {{count}}\nEpisodes: {{items}}",
|
||
"OUTPUT_NAME": "train-split.txt",
|
||
},
|
||
None,
|
||
),
|
||
# vla-model-train only reads JSON/JSONL/CSV samples, not this dataset's Parquet, so the
|
||
# VLA chain stops at the manifest check.
|
||
("vla_samples", "operator-builtin-vla-sample-manifest-validate", "VLA 样本清单校验", {"SOURCE_PATH": f"{STAGING_ROOT}/data", "CHECK_IMAGE_EXISTS": "false", "MAX_ROWS": "2000"}, None),
|
||
],
|
||
# The platform requires one connected graph, so the independent validators fan out from
|
||
# the split node; each reads its explicit SOURCE_PATH ahead of the upstream artifact.
|
||
"edges": [
|
||
("split", "agibot", "out-1", "in-1"),
|
||
("split", "rtx", "out-1", "in-1"),
|
||
("split", "v3", "out-1", "in-1"),
|
||
("split", "provenance", "out-1", "in-1"),
|
||
("split", "media_list", "out-1", "in-1"),
|
||
("media_list", "media", "out-1", "in-1"),
|
||
("split", "vla_samples", "out-1", "in-1"),
|
||
("split", "leakage", "out-1", "in-1"),
|
||
# Convert episode records, not the leakage validator's one-row report.
|
||
("split", "to_csv", "out-1", "in-1"),
|
||
("to_csv", "to_json", "out-1", "in-1"),
|
||
("to_json", "route", "out-1", "in-1"),
|
||
("route", "render", "matched", "in-1"),
|
||
],
|
||
},
|
||
"probe_schema": {
|
||
"name": "Ego 诊断·Parquet 校验单节点",
|
||
"description": f"诊断用:直接对已暂存的 {STAGING_ROOT} 运行 Parquet 结构校验(单节点)。",
|
||
"nodes": [
|
||
(
|
||
"schema",
|
||
"operator-builtin-lerobot-dataset-validate",
|
||
"Parquet 结构与索引校验",
|
||
{
|
||
"SOURCE_PATH": STAGING_ROOT,
|
||
"REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state",
|
||
"__image": "@runner",
|
||
},
|
||
None,
|
||
),
|
||
],
|
||
"edges": [],
|
||
},
|
||
}
|
||
|
||
|
||
def presigned_files(api: Api, sub_prefix: str) -> list[dict[str, Any]]:
|
||
"""Presigned (1h) download URLs for every object under DATASET_PREFIX/sub_prefix."""
|
||
manifest = api.request(
|
||
"GET",
|
||
"/api/object-storage/directory/manifest",
|
||
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": f"{DATASET_PREFIX}/{sub_prefix}".rstrip("/")},
|
||
)
|
||
return manifest.get("files") or []
|
||
|
||
|
||
def presigned_url(api: Api, relative_key: str) -> str:
|
||
directory, _, name = relative_key.rpartition("/")
|
||
for item in presigned_files(api, directory):
|
||
if item.get("key") == f"{DATASET_PREFIX}/{relative_key}":
|
||
return item["downloadUrl"]
|
||
raise RuntimeError(f"object not found: {DATASET_URI}/{relative_key}")
|
||
|
||
|
||
def items_payload(api: Api, sub_prefix: str, glob: str) -> dict[str, Any]:
|
||
"""Inline run input: one item per object, downloadable by video operators via source.previewURL."""
|
||
items = []
|
||
for item in presigned_files(api, sub_prefix):
|
||
name = item["key"].rsplit("/", 1)[-1]
|
||
relative = item.get("relativePath") or name
|
||
if not fnmatch.fnmatch(relative, glob) or "_droboticflow/" in f"/{relative}":
|
||
continue
|
||
items.append(
|
||
{
|
||
"filename": name,
|
||
"relativePath": relative,
|
||
"sizeBytes": item.get("sizeBytes", 0),
|
||
"source": {"path": f"bos://{BUCKET}/{item['key']}", "previewURL": item["downloadUrl"]},
|
||
}
|
||
)
|
||
if not items:
|
||
raise RuntimeError(f"no objects match {DATASET_URI}/{sub_prefix}/{glob}")
|
||
return {"datasetType": "ego-video-list", "sourcePath": f"{DATASET_URI}/{sub_prefix}", "items": items}
|
||
|
||
|
||
def resolve_overrides(api: Api, overrides: dict[str, str]) -> dict[str, str]:
|
||
resolved = {}
|
||
for key, value in overrides.items():
|
||
if value == "@runner":
|
||
value = api.request("GET", "/worker-api/capacity").get("runnerImage") or ""
|
||
if not value:
|
||
raise RuntimeError("worker did not report its operator-runner image")
|
||
elif value.startswith("@url:"):
|
||
value = presigned_url(api, value[len("@url:") :])
|
||
elif value.startswith("@run-output:"):
|
||
workflow_key, _, node_id = value[len("@run-output:") :].partition(":")
|
||
run_id = load_state()["runs"].get(workflow_key)
|
||
if not run_id:
|
||
raise RuntimeError(f"run `{workflow_key}` first")
|
||
value = f"/workspace/system/runs/{run_id}/nodes/{node_id}/outputs"
|
||
resolved[key] = value
|
||
return resolved
|
||
|
||
|
||
def live_operators(api: Api) -> dict[str, dict[str, Any]]:
|
||
return {item["id"]: item for item in items_of(api.request("GET", "/api/operators"))}
|
||
|
||
|
||
def node_payload(operator: dict[str, Any], label: str, overrides: dict[str, str], pattern: str | None, index: int) -> dict[str, Any]:
|
||
hyperparams = []
|
||
seen = set()
|
||
for item in operator.get("hyperparams") or []:
|
||
entry = dict(item)
|
||
if entry.get("key") in overrides:
|
||
entry["value"] = overrides[entry["key"]]
|
||
seen.add(entry.get("key"))
|
||
hyperparams.append(entry)
|
||
for key, value in overrides.items():
|
||
if key not in seen:
|
||
hyperparams.append({"key": key, "value": value})
|
||
data: dict[str, Any] = {
|
||
"label": label,
|
||
"category": operator.get("category", ""),
|
||
"description": operator.get("description", ""),
|
||
"operatorId": operator["id"],
|
||
"operatorImageId": operator.get("imageId", ""),
|
||
"image": operator.get("image", ""),
|
||
"containerCommand": operator.get("containerCommand") or [],
|
||
"containerArgs": operator.get("containerArgs") or [],
|
||
"runtimeContract": operator.get("runtimeContract") or {},
|
||
"skillDefinition": operator.get("skillDefinition", ""),
|
||
"inputs": operator.get("inputs") or [],
|
||
"outputs": operator.get("outputs") or [],
|
||
"hyperparams": hyperparams,
|
||
}
|
||
resources = operator.get("resources") or {}
|
||
if resources.get("cpuLimit"):
|
||
data["cpu"] = resources["cpuLimit"]
|
||
if resources.get("memoryLimit"):
|
||
data["memory"] = resources["memoryLimit"]
|
||
if resources.get("gpuCount"):
|
||
data["gpu"] = resources["gpuCount"]
|
||
if pattern == "@output":
|
||
data.update({"connectionId": CONNECTION_ID, "bucket": BUCKET})
|
||
elif pattern is not None:
|
||
# Source nodes must name the storage connection; without it the worker
|
||
# treats bos:// outputs as local files (why earlier digua runs failed).
|
||
data.update({"connectionId": CONNECTION_ID, "bucket": BUCKET, "prefix": DATASET_PREFIX, "pattern": pattern})
|
||
for key, value in {
|
||
"DATA_SOURCE": "BOS",
|
||
"SOURCE_PATH": DATASET_URI,
|
||
"STORAGE_PATTERN": pattern,
|
||
"SCAN_MODE": "FULL",
|
||
}.items():
|
||
data["hyperparams"] = [h for h in data["hyperparams"] if h.get("key") != key] + [{"key": key, "value": value}]
|
||
return {"id": f"node-{label and index}-{operator['id'][-8:]}", "type": "operator", "data": data, "position": {"x": 360 * index, "y": 0}}
|
||
|
||
|
||
def cloned_operator(api: Api, reference: str, cache: dict[str, Any]) -> dict[str, Any]:
|
||
"""Turn a node of an existing workflow into an operator-shaped dict so its exact settings are reused."""
|
||
workflow_id, _, node_id = reference.removeprefix("clone:").partition("#")
|
||
if workflow_id not in cache:
|
||
cache[workflow_id] = api.request("GET", f"/api/workflows/{workflow_id}")
|
||
node = next((n for n in cache[workflow_id].get("nodes", []) if n.get("id") == node_id), None)
|
||
if node is None:
|
||
raise RuntimeError(f"template node {node_id} not found in {workflow_id}")
|
||
data = dict(node["data"])
|
||
return {**data, "id": data["operatorId"], "imageId": data.get("operatorImageId", ""), "_clone": data}
|
||
|
||
|
||
def workflow_payload(spec: dict[str, Any], operators: dict[str, dict[str, Any]], api: Api | None = None) -> dict[str, Any]:
|
||
nodes = []
|
||
ids: dict[str, str] = {}
|
||
ports: dict[str, dict[str, Any]] = {}
|
||
clone_cache: dict[str, Any] = {}
|
||
for index, (key, operator_id, label, overrides, pattern) in enumerate(spec["nodes"]):
|
||
if operator_id == "@stage":
|
||
operator_id = load_state().get("stageOperatorId") or ""
|
||
if not operator_id:
|
||
raise RuntimeError("run `register-stage` first")
|
||
if operator_id.startswith("clone:"):
|
||
if api is None:
|
||
raise RuntimeError("cloning template nodes requires an API session")
|
||
operator = cloned_operator(api, operator_id, clone_cache)
|
||
else:
|
||
operator = operators.get(operator_id)
|
||
if operator is None:
|
||
raise RuntimeError(f"operator {operator_id} is not available in this organization")
|
||
if api is not None:
|
||
overrides = resolve_overrides(api, overrides)
|
||
image = overrides.pop("__image", None) if "__image" in overrides else None
|
||
node = node_payload(operator, label, overrides, pattern, index)
|
||
if image:
|
||
node["data"]["image"] = image
|
||
node["data"]["operatorImageId"] = image
|
||
if "_clone" in operator:
|
||
# Keep every template field (cpu/memory/gpu, image, args) and only change what we override.
|
||
cloned = {**operator["_clone"], "label": label, "hyperparams": node["data"]["hyperparams"]}
|
||
for field in ("connectionId", "bucket", "prefix", "pattern"):
|
||
if field in node["data"]:
|
||
cloned[field] = node["data"][field]
|
||
node["data"] = cloned
|
||
node["id"] = f"node-{key}"
|
||
ids[key] = node["id"]
|
||
ports[key] = operator
|
||
nodes.append(node)
|
||
edges = []
|
||
for source, target, *handles in spec["edges"]:
|
||
source_handle = handles[0] if handles else (ports[source].get("outputs") or [{}])[0].get("id", "out-1")
|
||
target_handle = handles[1] if len(handles) > 1 else (ports[target].get("inputs") or [{}])[0].get("id", "in-1")
|
||
edges.append(
|
||
{
|
||
"id": f"edge-{source}-{target}-{target_handle}",
|
||
"source": ids[source],
|
||
"sourceHandle": source_handle,
|
||
"target": ids[target],
|
||
"targetHandle": target_handle,
|
||
}
|
||
)
|
||
payload = {"name": spec["name"], "description": spec["description"], "type": "batch", "nodes": nodes, "edges": edges}
|
||
if spec.get("inputs"):
|
||
payload["workflowInputs"] = [
|
||
{
|
||
"id": item["id"],
|
||
"name": item["name"],
|
||
"type": "json",
|
||
"required": True,
|
||
"bindings": [{"nodeId": ids[item["node"]], "targetPortId": item["port"]}],
|
||
}
|
||
for item in spec["inputs"]
|
||
]
|
||
return payload
|
||
|
||
|
||
def run_inputs(api: Api, spec: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
item["id"]: {"mode": "inline", "type": "json", "inline": items_payload(api, *item["items"])}
|
||
for item in spec.get("inputs") or []
|
||
}
|
||
|
||
|
||
def load_state() -> dict[str, Any]:
|
||
path = STATE_DIR / "state.json"
|
||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"workflows": {}, "runs": {}}
|
||
|
||
|
||
def ensure_project(api: Api, state: dict[str, Any]) -> str:
|
||
if state.get("projectId"):
|
||
return state["projectId"]
|
||
for project in items_of(api.request("GET", "/api/projects")):
|
||
if project.get("name") == PROJECT_NAME:
|
||
state["projectId"] = project["id"]
|
||
return project["id"]
|
||
project = api.request(
|
||
"POST",
|
||
"/api/projects",
|
||
{"name": PROJECT_NAME, "description": f"第一视角(ego)数据处理:{DATASET_URI}", "status": "active"},
|
||
)
|
||
state["projectId"] = project["id"]
|
||
return project["id"]
|
||
|
||
|
||
def cmd_create(api: Api, keys: list[str], run: bool) -> None:
|
||
api.ensure_login()
|
||
resolve_connection_id(api)
|
||
state = load_state()
|
||
project_id = ensure_project(api, state)
|
||
operators = live_operators(api)
|
||
for key in keys or list(WORKFLOWS):
|
||
payload = workflow_payload(WORKFLOWS[key], operators, api)
|
||
existing = state["workflows"].get(key)
|
||
if existing:
|
||
workflow = api.request("PUT", f"/api/workflows/{existing}", payload)
|
||
print(f"updated workflow [{key}] {workflow['name']} {workflow['id']}")
|
||
else:
|
||
workflow = api.request("POST", f"/api/projects/{project_id}/workflows", payload)
|
||
state["workflows"][key] = workflow["id"]
|
||
print(f"created workflow [{key}] {workflow['name']} {workflow['id']}")
|
||
save("state.json", state)
|
||
if run:
|
||
body: dict[str, Any] = {"trigger": "manual"}
|
||
inputs = run_inputs(api, WORKFLOWS[key])
|
||
if inputs:
|
||
body["inputs"] = inputs
|
||
queued = api.request("POST", f"/api/workflows/{workflow['id']}/run", body)
|
||
state["runs"][key] = queued["id"]
|
||
save("state.json", state)
|
||
print(f" queued run {queued['id']}")
|
||
print(f"project: {PROJECT_NAME} ({project_id}) {API_BASE_URL}/projects/{project_id}")
|
||
|
||
|
||
def cmd_status(api: Api, keys: list[str], wait: bool) -> None:
|
||
api.ensure_login()
|
||
state = load_state()
|
||
targets = keys or list(state["runs"])
|
||
terminal = {"completed", "failed", "cancelled"}
|
||
while True:
|
||
report: dict[str, Any] = {}
|
||
for key in targets:
|
||
run_id = state["runs"].get(key)
|
||
if not run_id:
|
||
continue
|
||
run = api.request("GET", f"/api/runs/{run_id}")
|
||
entry = {"runId": run_id, "status": run.get("status"), "nodes": [], "raw": {k: v for k, v in run.items() if k not in ("logs", "nodeExecutions", "artifacts", "resourceSeries")}}
|
||
for node in run.get("nodeExecutions") or []:
|
||
entry["nodes"].append(
|
||
{"nodeId": node.get("nodeId"), "status": node.get("status"), "error": str(node.get("error") or "")[:1500]}
|
||
)
|
||
if run.get("status") in terminal:
|
||
try:
|
||
entry["artifacts"] = api.request("GET", f"/api/runs/{run_id}/artifacts")
|
||
except RuntimeError as error:
|
||
entry["artifacts"] = f"ERROR: {error}"
|
||
report[key] = entry
|
||
save("status.json", report)
|
||
stamp = time.strftime("%H:%M:%S")
|
||
for key, entry in report.items():
|
||
nodes = " ".join(f"{n['nodeId'].removeprefix('node-')}={n['status']}" for n in entry["nodes"])
|
||
print(f"[{stamp}] {key}: {entry['status']} {nodes}")
|
||
if not wait or all(entry["status"] in terminal for entry in report.values()):
|
||
return
|
||
time.sleep(20)
|
||
|
||
|
||
|
||
def cmd_show(api: Api) -> None:
|
||
"""Read the saved workflows back from the server; * marks nodes bound to a storage connection."""
|
||
api.ensure_login()
|
||
state = load_state()
|
||
for key, workflow_id in state["workflows"].items():
|
||
workflow = api.request("GET", f"/api/workflows/{workflow_id}")
|
||
nodes = [n["id"].removeprefix("node-") + ("*" if n["data"].get("connectionId") else "") for n in workflow["nodes"]]
|
||
print(f"[{key}] {workflow['name']} status={workflow.get('status')} edges={len(workflow['edges'])} nodes={nodes}")
|
||
|
||
|
||
def cmd_logs(api: Api, keys: list[str]) -> None:
|
||
"""Save run logs; keys are workflow keys from state.json or raw run ids."""
|
||
api.ensure_login()
|
||
state = load_state()
|
||
result = {}
|
||
for key in keys:
|
||
run_id = state["runs"].get(key, key)
|
||
result[key] = api.request("GET", f"/api/runs/{run_id}/logs")
|
||
print("saved:", save("logs.json", result))
|
||
|
||
|
||
def cmd_worker(api: Api) -> None:
|
||
"""Save worker stats/capacity to diagnose runs stuck in the queue."""
|
||
api.ensure_login()
|
||
result = {}
|
||
try:
|
||
settings = api.request("GET", "/api/settings")
|
||
result["workerSettings"] = settings.get("worker") if isinstance(settings, dict) else settings
|
||
except RuntimeError as error:
|
||
result["workerSettings"] = f"ERROR: {error}"
|
||
runs = items_of(api.request("GET", "/api/runs"))
|
||
result["activeRuns"] = [r for r in runs if r.get("status") in ("running", "queued", "stopping")]
|
||
for path in ("/worker-api/stats", "/worker-api/capacity"):
|
||
try:
|
||
result[path] = api.request("GET", path)
|
||
except RuntimeError as error:
|
||
result[path] = f"ERROR: {error}"
|
||
print("saved:", save("worker.json", result))
|
||
|
||
|
||
def cmd_artifacts(api: Api, keys: list[str]) -> None:
|
||
"""Save each run's artifacts plus the content of its JSON reports."""
|
||
api.ensure_login()
|
||
state = load_state()
|
||
result: dict[str, Any] = {}
|
||
for key in keys:
|
||
run_id = state["runs"].get(key, key)
|
||
artifacts = items_of(api.request("GET", f"/api/runs/{run_id}/artifacts"))
|
||
entries = []
|
||
for artifact in artifacts:
|
||
entry = dict(artifact)
|
||
if str(artifact.get("name", "")).endswith(".json"):
|
||
try:
|
||
entry["content"] = api.request("GET", f"/api/runs/{run_id}/artifacts/{artifact['id']}/content")
|
||
except RuntimeError as error:
|
||
entry["content"] = f"ERROR: {error}"
|
||
entries.append(entry)
|
||
result[key] = {"runId": run_id, "artifacts": entries}
|
||
print(f"{key}: {len(entries)} artifacts")
|
||
print("saved:", save("artifacts.json", result))
|
||
|
||
|
||
def cmd_stop(api: Api, keys: list[str], force: bool) -> None:
|
||
api.ensure_login()
|
||
state = load_state()
|
||
for key in keys:
|
||
run_id = state["runs"].get(key, key)
|
||
action = "force-stop" if force else "stop"
|
||
result = api.request("POST", f"/api/runs/{run_id}/{action}", {})
|
||
status = result.get("status") if isinstance(result, dict) else str(result)[:80]
|
||
print(f"{key}: {action} -> {status}")
|
||
|
||
|
||
STAGE_OPERATOR_NAME = "Ego 数据暂存(预签名下载到工作区)"
|
||
STAGE_CODE_SUBPATH = "lightweight-operators/ego-stage"
|
||
STAGE_MOUNT_PATH = "/opt/droboticflow/lightweight/ego-stage"
|
||
|
||
|
||
def ensure_stage_operator(api: Api) -> str:
|
||
"""Create (or refresh) the lightweight operator that stages presigned files onto the workspace PVC."""
|
||
state = load_state()
|
||
capacity = api.request("GET", "/worker-api/capacity")
|
||
image = capacity.get("runnerImage") or ""
|
||
if not image:
|
||
raise RuntimeError("worker did not report its operator-runner image")
|
||
payload = {
|
||
"name": STAGE_OPERATOR_NAME,
|
||
"type": "lightweight",
|
||
"category": "IO",
|
||
"description": "把工作流输入里的预签名 BOS 文件下载到当前运行的工作区目录,保持相对路径;tree 模式输出本地目录,files 模式另外逐个输出文件产物。",
|
||
"image": image,
|
||
# Not operator.py: that name would shadow the stdlib `operator` module.
|
||
"containerCommand": ["python", f"{STAGE_MOUNT_PATH}/ego_stage.py"],
|
||
"containerArgs": [],
|
||
"runtimeContract": {"lightweightOperator": {"codeSubPath": STAGE_CODE_SUBPATH, "mountPath": STAGE_MOUNT_PATH}},
|
||
# in-2 only lets a second branch join (collect mode ignores inputs).
|
||
"inputs": [{"id": "in-1", "name": "items", "type": "json"}, {"id": "in-2", "name": "join", "type": "json"}],
|
||
"outputs": [{"id": "out-1", "name": "output", "type": "json"}],
|
||
"hyperparams": [
|
||
{"key": "MODE", "value": "tree", "description": "tree=输出本地目录;files=另外逐个输出文件产物"},
|
||
{"key": "SUBDIR", "value": "dataset", "description": "输出目录下存放文件的子目录"},
|
||
],
|
||
"generationSource": "manual",
|
||
"publicationStatus": "published",
|
||
}
|
||
operator_id = state.get("stageOperatorId")
|
||
if operator_id:
|
||
api.request("PUT", f"/api/operators/{operator_id}", payload)
|
||
else:
|
||
operator_id = api.request("POST", "/api/operators", payload)["id"]
|
||
state["stageOperatorId"] = operator_id
|
||
save("state.json", state)
|
||
code = (ROOT / "lightweight-operators" / "ego-stage" / "ego_stage.py").read_text(encoding="utf-8")
|
||
api.request("PUT", f"/api/operators/{operator_id}/lightweight-files/content", {"path": "ego_stage.py", "content": code})
|
||
try:
|
||
api.request("DELETE", f"/api/operators/{operator_id}/lightweight-files", query={"path": "operator.py"})
|
||
except RuntimeError:
|
||
pass # Only present from the first registration.
|
||
return operator_id
|
||
|
||
|
||
def cmd_register_stage(api: Api) -> None:
|
||
api.ensure_login()
|
||
print("stage operator:", ensure_stage_operator(api))
|
||
|
||
|
||
def cmd_run_detail(api: Api, run_ids: list[str]) -> None:
|
||
"""Save full run records (node executions with inputs/outputs) for comparison."""
|
||
api.ensure_login()
|
||
state = load_state()
|
||
result = {key: api.request("GET", f"/api/runs/{state['runs'].get(key, key)}") for key in run_ids}
|
||
print("saved:", save("run-detail.json", result))
|
||
|
||
|
||
def cmd_download(api: Api, keys: list[str]) -> None:
|
||
"""Download objects (full keys in BUCKET) to output/ego-workflows/downloads/ via presigned URLs."""
|
||
api.ensure_login()
|
||
target_dir = STATE_DIR / "downloads"
|
||
target_dir.mkdir(parents=True, exist_ok=True)
|
||
for key in keys:
|
||
key = key.removeprefix(f"bos://{BUCKET}/")
|
||
directory = key.rsplit("/", 1)[0]
|
||
manifest = api.request(
|
||
"GET",
|
||
"/api/object-storage/directory/manifest",
|
||
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": directory},
|
||
)
|
||
match = next((item for item in manifest.get("files") or [] if item.get("key") == key), None)
|
||
if match is None:
|
||
raise RuntimeError(f"object not found: {key}")
|
||
target = target_dir / key.rsplit("/", 1)[-1]
|
||
with urllib.request.urlopen(match["downloadUrl"], timeout=600) as response, target.open("wb") as handle:
|
||
handle.write(response.read())
|
||
print(f"downloaded {target} ({target.stat().st_size} bytes)")
|
||
|
||
|
||
def cmd_pin_operator(api: Api, refs: list[str]) -> None:
|
||
"""Repin builtin operators to image@sha256 refs (operator id derived from the repository name)."""
|
||
api.ensure_login()
|
||
for ref in refs:
|
||
repository, _, digest = ref.partition("@")
|
||
if not digest.startswith("sha256:"):
|
||
raise RuntimeError(f"expected image@sha256:..., got {ref}")
|
||
operator_id = "operator-builtin-" + repository.rsplit("/", 1)[-1].removeprefix("droboticflow-operator-")
|
||
before = api.request("GET", "/api/operators").get("items", [])
|
||
previous = next((item.get("imageId") for item in before if item.get("id") == operator_id), None)
|
||
updated = api.request("PUT", f"/api/operators/{operator_id}", {"image": f"{repository}:builtin", "imageId": ref})
|
||
print(f"{operator_id}: {previous} -> {updated.get('imageId')}")
|
||
|
||
|
||
def cmd_probe(api: Api) -> None:
|
||
api.ensure_login()
|
||
connections = api.request("GET", "/api/storage/connections")
|
||
result: dict[str, Any] = {"connections": connections, "byConnection": {}}
|
||
items = connections.get("items", connections) if isinstance(connections, dict) else connections
|
||
for connection in items or []:
|
||
connection_id = connection.get("id", "")
|
||
entry: dict[str, Any] = {}
|
||
for label, path, query in [
|
||
("overview", "/api/object-storage", {"connectionId": connection_id}),
|
||
("root", "/api/object-storage/objects", {"connectionId": connection_id, "bucket": BUCKET, "prefix": "", "limit": 200}),
|
||
("dataset", "/api/object-storage/objects", {"connectionId": connection_id, "bucket": BUCKET, "prefix": DATASET_PREFIX + "/", "limit": 200}),
|
||
]:
|
||
try:
|
||
entry[label] = api.request("GET", path, query=query)
|
||
except RuntimeError as error:
|
||
entry[label] = f"ERROR: {error}"
|
||
result["byConnection"][connection_id] = entry
|
||
print("connections:", len(items or []))
|
||
print("saved:", save("probe.json", result))
|
||
|
||
|
||
def cmd_cat(api: Api, keys: list[str]) -> None:
|
||
api.ensure_login()
|
||
result = {}
|
||
for key in keys:
|
||
full = key if key.startswith(DATASET_PREFIX + "/") else f"{DATASET_PREFIX}/{key}"
|
||
result[full] = read_text_object(api, full)
|
||
print("saved:", save("cat.json", result))
|
||
|
||
|
||
def items_of(payload: Any) -> list[dict[str, Any]]:
|
||
if isinstance(payload, dict):
|
||
return payload.get("items") or []
|
||
return payload or []
|
||
|
||
|
||
def cmd_existing(api: Api) -> None:
|
||
"""Dump the org's workflows that touch the dataset, plus their latest runs."""
|
||
api.ensure_login()
|
||
matches = []
|
||
projects = items_of(api.request("GET", "/api/projects"))
|
||
for project in projects:
|
||
for summary in items_of(api.request("GET", f"/api/projects/{project['id']}/workflows")):
|
||
workflow = api.request("GET", f"/api/workflows/{summary['id']}")
|
||
text = json.dumps(workflow, ensure_ascii=False)
|
||
if "digua_test" not in text and "ego" not in text.lower():
|
||
continue
|
||
runs = items_of(api.request("GET", f"/api/workflows/{summary['id']}/runs"))[:3]
|
||
matches.append({"project": project.get("name"), "workflow": workflow, "runs": runs})
|
||
print(f"projects: {len(projects)}, matching workflows: {len(matches)}")
|
||
for match in matches:
|
||
statuses = [run.get("status") for run in match["runs"]]
|
||
print(f" [{match['project']}] {match['workflow'].get('name')} runs={statuses}")
|
||
print("saved:", save("existing.json", {"projects": projects, "matches": matches}))
|
||
|
||
|
||
def cmd_details(api: Api, run_ids: list[str]) -> None:
|
||
"""Save run details (status, node errors) plus the live operator catalog."""
|
||
api.ensure_login()
|
||
result: dict[str, Any] = {"runs": {}, "operators": api.request("GET", "/api/operators")}
|
||
for run_id in run_ids:
|
||
try:
|
||
result["runs"][run_id] = api.request("GET", f"/api/runs/{run_id}")
|
||
except RuntimeError as error:
|
||
result["runs"][run_id] = f"ERROR: {error}"
|
||
print("operators:", len(items_of(result["operators"])), "runs:", len(result["runs"]))
|
||
print("saved:", save("details.json", result))
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
parser.add_argument("command", choices=["inspect", "probe", "cat", "existing", "details", "create", "status", "show", "logs", "worker", "artifacts", "stop", "register-stage", "run-detail", "download", "pin-operator"])
|
||
parser.add_argument("keys", nargs="*", help="object keys (cat), run ids (details) or workflow keys (create/status)")
|
||
parser.add_argument("--run", action="store_true", help="create: start a run after saving each workflow")
|
||
parser.add_argument("--wait", action="store_true", help="status: poll until every run finishes")
|
||
parser.add_argument("--force", action="store_true", help="stop: force-stop instead of a graceful stop")
|
||
args = parser.parse_args()
|
||
api = Api()
|
||
if args.command == "inspect":
|
||
cmd_inspect(api)
|
||
elif args.command == "probe":
|
||
cmd_probe(api)
|
||
elif args.command == "existing":
|
||
cmd_existing(api)
|
||
elif args.command == "create":
|
||
cmd_create(api, args.keys, args.run)
|
||
elif args.command == "pin-operator":
|
||
cmd_pin_operator(api, args.keys)
|
||
elif args.command == "download":
|
||
cmd_download(api, args.keys)
|
||
elif args.command == "run-detail":
|
||
cmd_run_detail(api, args.keys)
|
||
elif args.command == "register-stage":
|
||
cmd_register_stage(api)
|
||
elif args.command == "stop":
|
||
cmd_stop(api, args.keys, args.force)
|
||
elif args.command == "artifacts":
|
||
cmd_artifacts(api, args.keys)
|
||
elif args.command == "worker":
|
||
cmd_worker(api)
|
||
elif args.command == "logs":
|
||
cmd_logs(api, args.keys)
|
||
elif args.command == "show":
|
||
cmd_show(api)
|
||
elif args.command == "status":
|
||
cmd_status(api, args.keys, args.wait)
|
||
elif args.command == "details":
|
||
cmd_details(api, args.keys)
|
||
elif args.command == "cat":
|
||
cmd_cat(api, args.keys)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
sys.exit(main())
|
||
except RuntimeError as error:
|
||
print("ERROR:", error, file=sys.stderr)
|
||
sys.exit(1)
|