droboticflow/scripts/ego_workflows.py
eust-w f00e9b9506 ✨ Stage ego data on the workspace PVC and split workflows for the POC worker
Add the ego-stage lightweight operator: it downloads presigned BOS files
into the run (or a stable /workspace/ego-staging path), collects another
node's outputs, and registers files as artifacts with ids. Operators such
as SAM3.1 and hand removal only read local paths, and their own images
emit artifacts without ids, which the current worker rejects.

Rework scripts/ego_workflows.py around what the current worker runs
reliably: short chains, one consumer per stage, validators on the
operator-runner image, plus commands to register the stage operator,
stop runs, read logs/run details and download results.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 15:03:12 +08:00

933 lines
42 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Inspect an egocentric LeRobot dataset on BOS and build processing workflows for it.
Usage:
python3 scripts/ego_workflows.py inspect
python3 scripts/ego_workflows.py create [KEY ...] [--run]
python3 scripts/ego_workflows.py status [KEY ...] [--wait]
Login: DROBOTICFLOW_EMAIL / DROBOTICFLOW_PASSWORD, otherwise prompted. The
session cookie is kept in output/ego-workflows/ so later steps reuse it.
"""
from __future__ import annotations
import argparse
import fnmatch
import getpass
import http.client
import http.cookiejar
import json
import os
import pathlib
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
ROOT = pathlib.Path(__file__).resolve().parents[1]
STATE_DIR = ROOT / "output" / "ego-workflows"
COOKIE_FILE = STATE_DIR / "cookies.txt"
API_BASE_URL = os.environ.get("API_BASE_URL", "http://120.48.164.19").rstrip("/")
BUCKET = os.environ.get("EGO_BUCKET", "testxyz")
DATASET_PREFIX = os.environ.get("EGO_DATASET_PREFIX", "digua_test/R16to9_10_sample").strip("/")
CAMERA_KEY = os.environ.get("EGO_CAMERA_KEY", "observation.images.cam_head_ego_left")
PROJECT_NAME = os.environ.get("EGO_PROJECT_NAME", "Ego 数据处理")
CONNECTION_ID = os.environ.get("EGO_CONNECTION_ID", "")
class Api:
def __init__(self) -> None:
STATE_DIR.mkdir(parents=True, exist_ok=True)
self.jar = http.cookiejar.MozillaCookieJar(str(COOKIE_FILE))
if COOKIE_FILE.exists():
self.jar.load(ignore_discard=True, ignore_expires=True)
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.jar))
def _csrf(self) -> str:
for cookie in self.jar:
if cookie.name == "droboticflow_csrf":
return cookie.value or ""
return ""
def request(self, method: str, path: str, body: Any = None, query: dict[str, Any] | None = None) -> Any:
url = API_BASE_URL + path
if query:
url += "?" + urllib.parse.urlencode({k: v for k, v in query.items() if v not in (None, "")})
data = None
headers = {"Accept": "application/json"}
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
headers["Content-Type"] = "application/json"
if method not in {"GET", "HEAD"} and self._csrf():
headers["X-Droboticflow-CSRF-Token"] = self._csrf()
request = urllib.request.Request(url, data=data, method=method, headers=headers)
try:
with self.opener.open(request, timeout=120) as response:
try:
raw = response.read()
except http.client.IncompleteRead as error:
raw = error.partial # the stop endpoints sometimes cut the chunked body short
except urllib.error.HTTPError as error:
detail = error.read().decode("utf-8", "replace")[:800]
raise RuntimeError(f"{method} {path} -> HTTP {error.code}: {detail}") from None
finally:
self.jar.save(ignore_discard=True, ignore_expires=True)
if not raw:
return None
try:
return json.loads(raw)
except json.JSONDecodeError:
return raw.decode("utf-8", "replace")
def ensure_login(self) -> dict[str, Any]:
try:
me = self.request("GET", "/api/auth/me")
if isinstance(me, dict) and (me.get("user") or me.get("id")):
return me
except RuntimeError:
pass
email = os.environ.get("DROBOTICFLOW_EMAIL") or input("Droboticflow 登录邮箱: ").strip()
password = os.environ.get("DROBOTICFLOW_PASSWORD") or getpass.getpass("密码: ")
self.request("POST", "/api/auth/login", {"email": email, "password": password})
return self.request("GET", "/api/auth/me")
def save(name: str, payload: Any) -> pathlib.Path:
path = STATE_DIR / name
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def resolve_connection_id(api: Api) -> str:
"""Pick the storage connection that exposes BUCKET (the org default may be unconfigured)."""
global CONNECTION_ID
if CONNECTION_ID:
return CONNECTION_ID
connections = api.request("GET", "/api/storage/connections")
items = connections.get("items", connections) if isinstance(connections, dict) else connections
for connection in items or []:
if connection.get("bucket") == BUCKET:
CONNECTION_ID = connection["id"]
return CONNECTION_ID
for connection in items or []:
overview = api.request("GET", "/api/object-storage", query={"connectionId": connection["id"]})
if BUCKET in [bucket.get("name") for bucket in overview.get("buckets") or []]:
CONNECTION_ID = connection["id"]
return CONNECTION_ID
raise RuntimeError(f"no storage connection exposes bucket {BUCKET}")
def list_directory(api: Api, prefix: str) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
marker = ""
while True:
page = api.request(
"GET",
"/api/object-storage/objects",
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": prefix, "marker": marker, "limit": 1000},
)
items.extend(page.get("items") or [])
marker = page.get("nextMarker") or ""
if not page.get("hasMore") or not marker:
return items
def list_objects(api: Api, prefix: str, limit: int = 20000) -> list[dict[str, Any]]:
"""Walk the directory-style listing into a flat list of files."""
files: list[dict[str, Any]] = []
pending = [prefix.rstrip("/") + "/"]
while pending and len(files) < limit:
current = pending.pop(0)
for item in list_directory(api, current):
if item.get("kind") == "directory":
pending.append(str(item["key"]).rstrip("/") + "/")
else:
files.append(item)
return files
def read_text_object(api: Api, key: str) -> str:
content = api.request(
"GET", "/api/object-storage/content", query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "key": key}
)
return content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
def cmd_inspect(api: Api) -> None:
me = api.ensure_login()
print("logged in:", json.dumps(me, ensure_ascii=False)[:200])
objects = list_objects(api, DATASET_PREFIX + "/")
keys = [str(item.get("key") or item.get("name") or "") for item in objects]
print(f"objects under bos://{BUCKET}/{DATASET_PREFIX}/: {len(keys)}")
meta: dict[str, Any] = {}
for key in keys:
if key.endswith((".json", ".jsonl")) and "/meta/" in f"/{key}" and len(meta) < 12:
try:
meta[key] = read_text_object(api, key)[:20000]
except RuntimeError as error:
meta[key] = f"ERROR: {error}"
path = save(
"inspect.json",
{"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": DATASET_PREFIX, "objects": objects, "meta": meta},
)
dirs: dict[str, int] = {}
for key in keys:
rel = key[len(DATASET_PREFIX) + 1 :] if key.startswith(DATASET_PREFIX + "/") else key
top = "/".join(rel.split("/")[:-1]) or "."
dirs[top] = dirs.get(top, 0) + 1
for name, count in sorted(dirs.items()):
print(f" {name}/ ({count} files)")
print("saved:", path)
# --- workflow definitions -------------------------------------------------
DATASET_URI = f"bos://{BUCKET}/{DATASET_PREFIX}"
OUTPUT_URI = os.environ.get("EGO_OUTPUT_URI", f"bos://{BUCKET}/{DATASET_PREFIX}_processed").rstrip("/")
LEFT_VIDEOS = f"videos/chunk-000/{CAMERA_KEY}"
SAMPLE_EPISODE = os.environ.get("EGO_SAMPLE_EPISODE", "episode_000000")
QC_GLOB = os.environ.get("EGO_QC_GLOB", "*.mp4")
# Proven egodex pipeline (SAM3.1 v6 -> hand removal v4 -> upload) whose node settings are cloned.
# Validator images from completed acceptance runs; the live catalog's `:builtin` digests never start.
SCHEMA_TEMPLATE = "workflow-23533841935b7e283183138b19825b0f#op-lerobot-dataset-validate"
METADATA_TEMPLATE = "workflow-602cefb887ee6c1f189c89e395f82b44#op-lerobot-metadata-consistency-validate"
ALIGNMENT_TEMPLATE = "workflow-602cefb887ee6c1f189c89e395f82b44#op-lerobot-video-alignment-validate"
# Stable staging dir on the workspace PVC for operators that only read local paths (SAM3, hand removal).
STAGING_ROOT = f"/workspace/ego-staging/{DATASET_PREFIX}"
SAMPLE_VIDEO_PATH = f"{STAGING_ROOT}/{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4"
MASK_DIR = f"/workspace/ego-staging/masks/{DATASET_PREFIX}/{CAMERA_KEY}"
HAND_REMOVAL_TEMPLATE = os.environ.get("EGO_HAND_REMOVAL_TEMPLATE", "workflow-a77bd54acf37717a2a72339021182072")
# Each node: (node id, operator id, label, hyperparam overrides, BOS source pattern or None).
# Edges connect each operator's first output to the next operator's first input.
WORKFLOWS: dict[str, dict[str, Any]] = {
"smoke": {
"name": "Ego 冒烟-元数据读取",
"description": f"读取 {DATASET_URI} 的 meta/ 与 data/(不含视频),生成 LeRobot 数据集概览,用于验证 BOS 数据接入链路。",
"nodes": [
("source", "operator-builtin-real-source", "Ego 元数据与 Parquet", {}, "[md][ea]ta/*"),
("summary", "operator-builtin-lerobot-summary", "LeRobot 数据集概览", {}, None),
],
"edges": [("source", "summary")],
},
# Each check is its own two-node workflow (stage -> one validator). On the current POC
# worker, a third node in a chain or parallel branches stall before their Kubernetes Job
# is created; stage -> one consumer runs reliably. Staged files are reused across runs.
**{
key: {
"name": f"Ego 数据集校验·{title}",
"description": f"把 {DATASET_URI} 的 meta/、data/、videos/ 暂存到工作区 {STAGING_ROOT}(已暂存的文件复用),再执行{title}。",
"nodes": [
("stage", "@stage", "数据集暂存到工作区", {"MODE": "tree", "SUBDIR": STAGING_ROOT}, None),
(node, operator, title, {"SOURCE_PATH": STAGING_ROOT, **overrides}, None),
],
"edges": [("stage", node, "out-1", "in-1")],
"inputs": [{"id": "dataset", "name": "数据集文件清单", "node": "stage", "port": "in-1", "items": ("", "[mdv]*/*")}],
}
for key, node, title, operator, overrides in [
("integrity", "summary", "LeRobot 数据集概览", "operator-builtin-lerobot-summary", {}),
# The validators' own images (catalog `:builtin` and the July builds) never finish a
# job on the current cluster; the worker's operator-runner image bundles the same
# handlers (operator-repos + pyarrow/opencv), so run them on that instead.
(
"integrity_schema",
"schema",
"Parquet 结构与索引校验",
"operator-builtin-lerobot-dataset-validate",
{"REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state", "__image": "@runner"},
),
(
"integrity_metadata",
"metadata",
"元数据一致性校验",
"operator-builtin-lerobot-metadata-consistency-validate",
{"REQUIRED_FEATURES": f"action,observation.state,left_kp3d,right_kp3d,{CAMERA_KEY}", "__image": "@runner"},
),
(
"integrity_alignment",
"alignment",
"视频帧对齐校验",
"operator-builtin-lerobot-video-alignment-validate",
{"FRAME_TOLERANCE": "1", "MAX_VIDEOS": "20", "__image": "@runner"},
),
]
},
"left_qc": {
"name": "Ego 左目视频质检",
"description": (
f"对 {LEFT_VIDEOS} 的全部视频计算清晰度、抖动与无手率,并按阈值判定通过/不通过。"
"视频清单在每次运行时作为工作流输入传入(含 1 小时有效的 BOS 预签名下载地址)。"
),
# No separate filter node: video-quality already marks each item passed/failed, and on
# the current worker a node consuming its externalized dataRef output never launches.
"nodes": [
("quality", "operator-builtin-video-quality", "视频质检(含通过/不通过判定)", {}, None),
],
"edges": [],
"inputs": [
{"id": "videos", "name": "左目视频清单", "node": "quality", "port": "in-1", "items": (LEFT_VIDEOS, QC_GLOB)},
],
},
"wilor": {
"name": f"Ego 手部 3D 姿态估计(WiLoR,{SAMPLE_EPISODE})",
"description": (
f"对左目 {SAMPLE_EPISODE}.mp4 抽帧调用 WiLoR 估计双手 3D 姿态,结果上传到 {OUTPUT_URI}/wilor/。"
"INPUT_URL 在每次运行前刷新为 1 小时有效的 BOS 预签名地址。"
),
"nodes": [
(
"source",
"operator-builtin-real-source",
"Ego 视频引用",
{"DATA_SOURCE": "INLINE", "SOURCE_JSON": json.dumps({"video": f"{DATASET_URI}/{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4"})},
None,
),
(
"wilor",
"operator-builtin-wilor-hand-pose",
"WiLoR 3D 手部姿态",
{
"INPUT_URL": f"@url:{LEFT_VIDEOS}/{SAMPLE_EPISODE}.mp4",
"INPUT_PATH": "",
"MEDIA_KIND": "video",
"FRAME_STRIDE": "3",
"MAX_FRAMES": "120",
},
None,
),
("upload", "operator-builtin-upload", "结果上传 BOS", {"OUTPUT_PATH": f"{OUTPUT_URI}/wilor/"}, "@output"),
],
"edges": [("source", "wilor"), ("wilor", "upload")],
},
# Hand removal is split in two for the same worker limitation. SAM3 and the hand-removal
# image only read local paths (artifacts arrive as bos:// refs), so the video and the masks
# go through stable workspace paths. Results are persisted by the worker's artifact storage.
"hand_removal": {
"name": f"Ego 去手①SAM3.1 手与手臂分割({SAMPLE_EPISODE})",
"description": f"把左目 {SAMPLE_EPISODE}.mp4 暂存到 {STAGING_ROOT},用 SAM3.1 分割手与手臂,掩码汇总到 {MASK_DIR}。",
"nodes": [
("source", "@stage", f"左目视频 {SAMPLE_EPISODE} 暂存", {"MODE": "tree", "SUBDIR": f"{STAGING_ROOT}/{LEFT_VIDEOS}"}, None),
("sam3", f"clone:{HAND_REMOVAL_TEMPLATE}#node-sam3-1", "SAM3.1 手与手臂分割", {"VIDEO_PATH": SAMPLE_VIDEO_PATH}, None),
("collect", "@stage", "SAM3 掩码汇总到工作区", {"MODE": "collect", "SOURCE_NODE": "node-sam3", "SUBDIR": MASK_DIR}, None),
],
"edges": [("source", "sam3", "out-1", "in-1"), ("sam3", "collect", "out-1", "in-1")],
"inputs": [
{"id": "video", "name": "左目视频", "node": "source", "port": "in-1", "items": (LEFT_VIDEOS, f"{SAMPLE_EPISODE}.mp4")},
],
},
"hand_removal_inpaint": {
"name": f"Ego 去手②ProPainter 去手({SAMPLE_EPISODE})",
"description": f"读取 {MASK_DIR} 中 SAM3.1 的掩码,对左目 {SAMPLE_EPISODE}.mp4 去手(需先运行 去手①)。",
"nodes": [
("source", "@stage", f"左目视频 {SAMPLE_EPISODE} 暂存", {"MODE": "tree", "SUBDIR": f"{STAGING_ROOT}/{LEFT_VIDEOS}"}, None),
(
"removal",
f"clone:{HAND_REMOVAL_TEMPLATE}#node-hr-1",
"人手去除",
{"INPUT_VIDEO": SAMPLE_VIDEO_PATH, "MASK_SOURCE": f"{MASK_DIR}/{SAMPLE_EPISODE}/track_result.json"},
None,
),
],
"edges": [("source", "removal", "out-1", "in-1")],
"inputs": [
{"id": "video", "name": "左目视频", "node": "source", "port": "in-1", "items": (LEFT_VIDEOS, f"{SAMPLE_EPISODE}.mp4")},
],
},
# The hand-removal image emits artifacts without ids, so the worker fails that node even
# though the cleaned video was written. This registers its outputs as proper artifacts.
"hand_removal_result": {
"name": f"Ego 去手③结果登记({SAMPLE_EPISODE})",
"description": "把 去手② 最近一次运行写在工作区的去手视频与摘要登记为产物(由平台持久化到 BOS 产物存储)。",
"nodes": [
(
"collect",
"@stage",
"去手结果登记",
{
"MODE": "collect",
"SOURCE_DIR": "@run-output:hand_removal_inpaint:node-removal",
"SUBDIR": f"/workspace/ego-staging/hand-removal/{DATASET_PREFIX}/{CAMERA_KEY}",
"EMIT_FILES": "true",
},
None,
),
],
"edges": [],
},
"probe_schema": {
"name": "Ego 诊断·Parquet 校验单节点",
"description": f"诊断用:直接对已暂存的 {STAGING_ROOT} 运行 Parquet 结构校验(单节点)。",
"nodes": [
(
"schema",
"operator-builtin-lerobot-dataset-validate",
"Parquet 结构与索引校验",
{
"SOURCE_PATH": STAGING_ROOT,
"REQUIRED_COLUMNS": "episode_index,frame_index,timestamp,action,observation.state",
"__image": "@runner",
},
None,
),
],
"edges": [],
},
}
def presigned_files(api: Api, sub_prefix: str) -> list[dict[str, Any]]:
"""Presigned (1h) download URLs for every object under DATASET_PREFIX/sub_prefix."""
manifest = api.request(
"GET",
"/api/object-storage/directory/manifest",
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": f"{DATASET_PREFIX}/{sub_prefix}".rstrip("/")},
)
return manifest.get("files") or []
def presigned_url(api: Api, relative_key: str) -> str:
directory, _, name = relative_key.rpartition("/")
for item in presigned_files(api, directory):
if item.get("key") == f"{DATASET_PREFIX}/{relative_key}":
return item["downloadUrl"]
raise RuntimeError(f"object not found: {DATASET_URI}/{relative_key}")
def items_payload(api: Api, sub_prefix: str, glob: str) -> dict[str, Any]:
"""Inline run input: one item per object, downloadable by video operators via source.previewURL."""
items = []
for item in presigned_files(api, sub_prefix):
name = item["key"].rsplit("/", 1)[-1]
relative = item.get("relativePath") or name
if not fnmatch.fnmatch(relative, glob) or "_droboticflow/" in f"/{relative}":
continue
items.append(
{
"filename": name,
"relativePath": relative,
"sizeBytes": item.get("sizeBytes", 0),
"source": {"path": f"bos://{BUCKET}/{item['key']}", "previewURL": item["downloadUrl"]},
}
)
if not items:
raise RuntimeError(f"no objects match {DATASET_URI}/{sub_prefix}/{glob}")
return {"datasetType": "ego-video-list", "sourcePath": f"{DATASET_URI}/{sub_prefix}", "items": items}
def resolve_overrides(api: Api, overrides: dict[str, str]) -> dict[str, str]:
resolved = {}
for key, value in overrides.items():
if value == "@runner":
value = api.request("GET", "/worker-api/capacity").get("runnerImage") or ""
if not value:
raise RuntimeError("worker did not report its operator-runner image")
elif value.startswith("@url:"):
value = presigned_url(api, value[len("@url:") :])
elif value.startswith("@run-output:"):
workflow_key, _, node_id = value[len("@run-output:") :].partition(":")
run_id = load_state()["runs"].get(workflow_key)
if not run_id:
raise RuntimeError(f"run `{workflow_key}` first")
value = f"/workspace/system/runs/{run_id}/nodes/{node_id}/outputs"
resolved[key] = value
return resolved
def live_operators(api: Api) -> dict[str, dict[str, Any]]:
return {item["id"]: item for item in items_of(api.request("GET", "/api/operators"))}
def node_payload(operator: dict[str, Any], label: str, overrides: dict[str, str], pattern: str | None, index: int) -> dict[str, Any]:
hyperparams = []
seen = set()
for item in operator.get("hyperparams") or []:
entry = dict(item)
if entry.get("key") in overrides:
entry["value"] = overrides[entry["key"]]
seen.add(entry.get("key"))
hyperparams.append(entry)
for key, value in overrides.items():
if key not in seen:
hyperparams.append({"key": key, "value": value})
data: dict[str, Any] = {
"label": label,
"category": operator.get("category", ""),
"description": operator.get("description", ""),
"operatorId": operator["id"],
"operatorImageId": operator.get("imageId", ""),
"image": operator.get("image", ""),
"containerCommand": operator.get("containerCommand") or [],
"containerArgs": operator.get("containerArgs") or [],
"runtimeContract": operator.get("runtimeContract") or {},
"skillDefinition": operator.get("skillDefinition", ""),
"inputs": operator.get("inputs") or [],
"outputs": operator.get("outputs") or [],
"hyperparams": hyperparams,
}
resources = operator.get("resources") or {}
if resources.get("cpuLimit"):
data["cpu"] = resources["cpuLimit"]
if resources.get("memoryLimit"):
data["memory"] = resources["memoryLimit"]
if resources.get("gpuCount"):
data["gpu"] = resources["gpuCount"]
if pattern == "@output":
data.update({"connectionId": CONNECTION_ID, "bucket": BUCKET})
elif pattern is not None:
# Source nodes must name the storage connection; without it the worker
# treats bos:// outputs as local files (why earlier digua runs failed).
data.update({"connectionId": CONNECTION_ID, "bucket": BUCKET, "prefix": DATASET_PREFIX, "pattern": pattern})
for key, value in {
"DATA_SOURCE": "BOS",
"SOURCE_PATH": DATASET_URI,
"STORAGE_PATTERN": pattern,
"SCAN_MODE": "FULL",
}.items():
data["hyperparams"] = [h for h in data["hyperparams"] if h.get("key") != key] + [{"key": key, "value": value}]
return {"id": f"node-{label and index}-{operator['id'][-8:]}", "type": "operator", "data": data, "position": {"x": 360 * index, "y": 0}}
def cloned_operator(api: Api, reference: str, cache: dict[str, Any]) -> dict[str, Any]:
"""Turn a node of an existing workflow into an operator-shaped dict so its exact settings are reused."""
workflow_id, _, node_id = reference.removeprefix("clone:").partition("#")
if workflow_id not in cache:
cache[workflow_id] = api.request("GET", f"/api/workflows/{workflow_id}")
node = next((n for n in cache[workflow_id].get("nodes", []) if n.get("id") == node_id), None)
if node is None:
raise RuntimeError(f"template node {node_id} not found in {workflow_id}")
data = dict(node["data"])
return {**data, "id": data["operatorId"], "imageId": data.get("operatorImageId", ""), "_clone": data}
def workflow_payload(spec: dict[str, Any], operators: dict[str, dict[str, Any]], api: Api | None = None) -> dict[str, Any]:
nodes = []
ids: dict[str, str] = {}
ports: dict[str, dict[str, Any]] = {}
clone_cache: dict[str, Any] = {}
for index, (key, operator_id, label, overrides, pattern) in enumerate(spec["nodes"]):
if operator_id == "@stage":
operator_id = load_state().get("stageOperatorId") or ""
if not operator_id:
raise RuntimeError("run `register-stage` first")
if operator_id.startswith("clone:"):
if api is None:
raise RuntimeError("cloning template nodes requires an API session")
operator = cloned_operator(api, operator_id, clone_cache)
else:
operator = operators.get(operator_id)
if operator is None:
raise RuntimeError(f"operator {operator_id} is not available in this organization")
if api is not None:
overrides = resolve_overrides(api, overrides)
image = overrides.pop("__image", None) if "__image" in overrides else None
node = node_payload(operator, label, overrides, pattern, index)
if image:
node["data"]["image"] = image
node["data"]["operatorImageId"] = image
if "_clone" in operator:
# Keep every template field (cpu/memory/gpu, image, args) and only change what we override.
cloned = {**operator["_clone"], "label": label, "hyperparams": node["data"]["hyperparams"]}
for field in ("connectionId", "bucket", "prefix", "pattern"):
if field in node["data"]:
cloned[field] = node["data"][field]
node["data"] = cloned
node["id"] = f"node-{key}"
ids[key] = node["id"]
ports[key] = operator
nodes.append(node)
edges = []
for source, target, *handles in spec["edges"]:
source_handle = handles[0] if handles else (ports[source].get("outputs") or [{}])[0].get("id", "out-1")
target_handle = handles[1] if len(handles) > 1 else (ports[target].get("inputs") or [{}])[0].get("id", "in-1")
edges.append(
{
"id": f"edge-{source}-{target}-{target_handle}",
"source": ids[source],
"sourceHandle": source_handle,
"target": ids[target],
"targetHandle": target_handle,
}
)
payload = {"name": spec["name"], "description": spec["description"], "type": "batch", "nodes": nodes, "edges": edges}
if spec.get("inputs"):
payload["workflowInputs"] = [
{
"id": item["id"],
"name": item["name"],
"type": "json",
"required": True,
"bindings": [{"nodeId": ids[item["node"]], "targetPortId": item["port"]}],
}
for item in spec["inputs"]
]
return payload
def run_inputs(api: Api, spec: dict[str, Any]) -> dict[str, Any]:
return {
item["id"]: {"mode": "inline", "type": "json", "inline": items_payload(api, *item["items"])}
for item in spec.get("inputs") or []
}
def load_state() -> dict[str, Any]:
path = STATE_DIR / "state.json"
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"workflows": {}, "runs": {}}
def ensure_project(api: Api, state: dict[str, Any]) -> str:
if state.get("projectId"):
return state["projectId"]
for project in items_of(api.request("GET", "/api/projects")):
if project.get("name") == PROJECT_NAME:
state["projectId"] = project["id"]
return project["id"]
project = api.request(
"POST",
"/api/projects",
{"name": PROJECT_NAME, "description": f"第一视角(ego)数据处理:{DATASET_URI}", "status": "active"},
)
state["projectId"] = project["id"]
return project["id"]
def cmd_create(api: Api, keys: list[str], run: bool) -> None:
api.ensure_login()
resolve_connection_id(api)
state = load_state()
project_id = ensure_project(api, state)
operators = live_operators(api)
for key in keys or list(WORKFLOWS):
payload = workflow_payload(WORKFLOWS[key], operators, api)
existing = state["workflows"].get(key)
if existing:
workflow = api.request("PUT", f"/api/workflows/{existing}", payload)
print(f"updated workflow [{key}] {workflow['name']} {workflow['id']}")
else:
workflow = api.request("POST", f"/api/projects/{project_id}/workflows", payload)
state["workflows"][key] = workflow["id"]
print(f"created workflow [{key}] {workflow['name']} {workflow['id']}")
save("state.json", state)
if run:
body: dict[str, Any] = {"trigger": "manual"}
inputs = run_inputs(api, WORKFLOWS[key])
if inputs:
body["inputs"] = inputs
queued = api.request("POST", f"/api/workflows/{workflow['id']}/run", body)
state["runs"][key] = queued["id"]
save("state.json", state)
print(f" queued run {queued['id']}")
print(f"project: {PROJECT_NAME} ({project_id}) {API_BASE_URL}/projects/{project_id}")
def cmd_status(api: Api, keys: list[str], wait: bool) -> None:
api.ensure_login()
state = load_state()
targets = keys or list(state["runs"])
terminal = {"completed", "failed", "cancelled"}
while True:
report: dict[str, Any] = {}
for key in targets:
run_id = state["runs"].get(key)
if not run_id:
continue
run = api.request("GET", f"/api/runs/{run_id}")
entry = {"runId": run_id, "status": run.get("status"), "nodes": [], "raw": {k: v for k, v in run.items() if k not in ("logs", "nodeExecutions", "artifacts", "resourceSeries")}}
for node in run.get("nodeExecutions") or []:
entry["nodes"].append(
{"nodeId": node.get("nodeId"), "status": node.get("status"), "error": str(node.get("error") or "")[:1500]}
)
if run.get("status") in terminal:
try:
entry["artifacts"] = api.request("GET", f"/api/runs/{run_id}/artifacts")
except RuntimeError as error:
entry["artifacts"] = f"ERROR: {error}"
report[key] = entry
save("status.json", report)
stamp = time.strftime("%H:%M:%S")
for key, entry in report.items():
nodes = " ".join(f"{n['nodeId'].removeprefix('node-')}={n['status']}" for n in entry["nodes"])
print(f"[{stamp}] {key}: {entry['status']} {nodes}")
if not wait or all(entry["status"] in terminal for entry in report.values()):
return
time.sleep(20)
def cmd_show(api: Api) -> None:
"""Read the saved workflows back from the server; * marks nodes bound to a storage connection."""
api.ensure_login()
state = load_state()
for key, workflow_id in state["workflows"].items():
workflow = api.request("GET", f"/api/workflows/{workflow_id}")
nodes = [n["id"].removeprefix("node-") + ("*" if n["data"].get("connectionId") else "") for n in workflow["nodes"]]
print(f"[{key}] {workflow['name']} status={workflow.get('status')} edges={len(workflow['edges'])} nodes={nodes}")
def cmd_logs(api: Api, keys: list[str]) -> None:
"""Save run logs; keys are workflow keys from state.json or raw run ids."""
api.ensure_login()
state = load_state()
result = {}
for key in keys:
run_id = state["runs"].get(key, key)
result[key] = api.request("GET", f"/api/runs/{run_id}/logs")
print("saved:", save("logs.json", result))
def cmd_worker(api: Api) -> None:
"""Save worker stats/capacity to diagnose runs stuck in the queue."""
api.ensure_login()
result = {}
try:
settings = api.request("GET", "/api/settings")
result["workerSettings"] = settings.get("worker") if isinstance(settings, dict) else settings
except RuntimeError as error:
result["workerSettings"] = f"ERROR: {error}"
runs = items_of(api.request("GET", "/api/runs"))
result["activeRuns"] = [r for r in runs if r.get("status") in ("running", "queued", "stopping")]
for path in ("/worker-api/stats", "/worker-api/capacity"):
try:
result[path] = api.request("GET", path)
except RuntimeError as error:
result[path] = f"ERROR: {error}"
print("saved:", save("worker.json", result))
def cmd_artifacts(api: Api, keys: list[str]) -> None:
"""Save each run's artifacts plus the content of its JSON reports."""
api.ensure_login()
state = load_state()
result: dict[str, Any] = {}
for key in keys:
run_id = state["runs"].get(key, key)
artifacts = items_of(api.request("GET", f"/api/runs/{run_id}/artifacts"))
entries = []
for artifact in artifacts:
entry = dict(artifact)
if str(artifact.get("name", "")).endswith(".json"):
try:
entry["content"] = api.request("GET", f"/api/runs/{run_id}/artifacts/{artifact['id']}/content")
except RuntimeError as error:
entry["content"] = f"ERROR: {error}"
entries.append(entry)
result[key] = {"runId": run_id, "artifacts": entries}
print(f"{key}: {len(entries)} artifacts")
print("saved:", save("artifacts.json", result))
def cmd_stop(api: Api, keys: list[str], force: bool) -> None:
api.ensure_login()
state = load_state()
for key in keys:
run_id = state["runs"].get(key, key)
action = "force-stop" if force else "stop"
result = api.request("POST", f"/api/runs/{run_id}/{action}", {})
status = result.get("status") if isinstance(result, dict) else str(result)[:80]
print(f"{key}: {action} -> {status}")
STAGE_OPERATOR_NAME = "Ego 数据暂存(预签名下载到工作区)"
STAGE_CODE_SUBPATH = "lightweight-operators/ego-stage"
STAGE_MOUNT_PATH = "/opt/droboticflow/lightweight/ego-stage"
def ensure_stage_operator(api: Api) -> str:
"""Create (or refresh) the lightweight operator that stages presigned files onto the workspace PVC."""
state = load_state()
capacity = api.request("GET", "/worker-api/capacity")
image = capacity.get("runnerImage") or ""
if not image:
raise RuntimeError("worker did not report its operator-runner image")
payload = {
"name": STAGE_OPERATOR_NAME,
"type": "lightweight",
"category": "IO",
"description": "把工作流输入里的预签名 BOS 文件下载到当前运行的工作区目录,保持相对路径;tree 模式输出本地目录,files 模式另外逐个输出文件产物。",
"image": image,
# Not operator.py: that name would shadow the stdlib `operator` module.
"containerCommand": ["python", f"{STAGE_MOUNT_PATH}/ego_stage.py"],
"containerArgs": [],
"runtimeContract": {"lightweightOperator": {"codeSubPath": STAGE_CODE_SUBPATH, "mountPath": STAGE_MOUNT_PATH}},
"inputs": [{"id": "in-1", "name": "items", "type": "json"}],
"outputs": [{"id": "out-1", "name": "output", "type": "json"}],
"hyperparams": [
{"key": "MODE", "value": "tree", "description": "tree=输出本地目录;files=另外逐个输出文件产物"},
{"key": "SUBDIR", "value": "dataset", "description": "输出目录下存放文件的子目录"},
],
"generationSource": "manual",
"publicationStatus": "published",
}
operator_id = state.get("stageOperatorId")
if operator_id:
api.request("PUT", f"/api/operators/{operator_id}", payload)
else:
operator_id = api.request("POST", "/api/operators", payload)["id"]
state["stageOperatorId"] = operator_id
save("state.json", state)
code = (ROOT / "lightweight-operators" / "ego-stage" / "ego_stage.py").read_text(encoding="utf-8")
api.request("PUT", f"/api/operators/{operator_id}/lightweight-files/content", {"path": "ego_stage.py", "content": code})
try:
api.request("DELETE", f"/api/operators/{operator_id}/lightweight-files", query={"path": "operator.py"})
except RuntimeError:
pass # Only present from the first registration.
return operator_id
def cmd_register_stage(api: Api) -> None:
api.ensure_login()
print("stage operator:", ensure_stage_operator(api))
def cmd_run_detail(api: Api, run_ids: list[str]) -> None:
"""Save full run records (node executions with inputs/outputs) for comparison."""
api.ensure_login()
state = load_state()
result = {key: api.request("GET", f"/api/runs/{state['runs'].get(key, key)}") for key in run_ids}
print("saved:", save("run-detail.json", result))
def cmd_download(api: Api, keys: list[str]) -> None:
"""Download objects (full keys in BUCKET) to output/ego-workflows/downloads/ via presigned URLs."""
api.ensure_login()
target_dir = STATE_DIR / "downloads"
target_dir.mkdir(parents=True, exist_ok=True)
for key in keys:
key = key.removeprefix(f"bos://{BUCKET}/")
directory = key.rsplit("/", 1)[0]
manifest = api.request(
"GET",
"/api/object-storage/directory/manifest",
query={"connectionId": resolve_connection_id(api), "bucket": BUCKET, "prefix": directory},
)
match = next((item for item in manifest.get("files") or [] if item.get("key") == key), None)
if match is None:
raise RuntimeError(f"object not found: {key}")
target = target_dir / key.rsplit("/", 1)[-1]
with urllib.request.urlopen(match["downloadUrl"], timeout=600) as response, target.open("wb") as handle:
handle.write(response.read())
print(f"downloaded {target} ({target.stat().st_size} bytes)")
def cmd_probe(api: Api) -> None:
api.ensure_login()
connections = api.request("GET", "/api/storage/connections")
result: dict[str, Any] = {"connections": connections, "byConnection": {}}
items = connections.get("items", connections) if isinstance(connections, dict) else connections
for connection in items or []:
connection_id = connection.get("id", "")
entry: dict[str, Any] = {}
for label, path, query in [
("overview", "/api/object-storage", {"connectionId": connection_id}),
("root", "/api/object-storage/objects", {"connectionId": connection_id, "bucket": BUCKET, "prefix": "", "limit": 200}),
("dataset", "/api/object-storage/objects", {"connectionId": connection_id, "bucket": BUCKET, "prefix": DATASET_PREFIX + "/", "limit": 200}),
]:
try:
entry[label] = api.request("GET", path, query=query)
except RuntimeError as error:
entry[label] = f"ERROR: {error}"
result["byConnection"][connection_id] = entry
print("connections:", len(items or []))
print("saved:", save("probe.json", result))
def cmd_cat(api: Api, keys: list[str]) -> None:
api.ensure_login()
result = {}
for key in keys:
full = key if key.startswith(DATASET_PREFIX + "/") else f"{DATASET_PREFIX}/{key}"
result[full] = read_text_object(api, full)
print("saved:", save("cat.json", result))
def items_of(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, dict):
return payload.get("items") or []
return payload or []
def cmd_existing(api: Api) -> None:
"""Dump the org's workflows that touch the dataset, plus their latest runs."""
api.ensure_login()
matches = []
projects = items_of(api.request("GET", "/api/projects"))
for project in projects:
for summary in items_of(api.request("GET", f"/api/projects/{project['id']}/workflows")):
workflow = api.request("GET", f"/api/workflows/{summary['id']}")
text = json.dumps(workflow, ensure_ascii=False)
if "digua_test" not in text and "ego" not in text.lower():
continue
runs = items_of(api.request("GET", f"/api/workflows/{summary['id']}/runs"))[:3]
matches.append({"project": project.get("name"), "workflow": workflow, "runs": runs})
print(f"projects: {len(projects)}, matching workflows: {len(matches)}")
for match in matches:
statuses = [run.get("status") for run in match["runs"]]
print(f" [{match['project']}] {match['workflow'].get('name')} runs={statuses}")
print("saved:", save("existing.json", {"projects": projects, "matches": matches}))
def cmd_details(api: Api, run_ids: list[str]) -> None:
"""Save run details (status, node errors) plus the live operator catalog."""
api.ensure_login()
result: dict[str, Any] = {"runs": {}, "operators": api.request("GET", "/api/operators")}
for run_id in run_ids:
try:
result["runs"][run_id] = api.request("GET", f"/api/runs/{run_id}")
except RuntimeError as error:
result["runs"][run_id] = f"ERROR: {error}"
print("operators:", len(items_of(result["operators"])), "runs:", len(result["runs"]))
print("saved:", save("details.json", result))
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("command", choices=["inspect", "probe", "cat", "existing", "details", "create", "status", "show", "logs", "worker", "artifacts", "stop", "register-stage", "run-detail", "download"])
parser.add_argument("keys", nargs="*", help="object keys (cat), run ids (details) or workflow keys (create/status)")
parser.add_argument("--run", action="store_true", help="create: start a run after saving each workflow")
parser.add_argument("--wait", action="store_true", help="status: poll until every run finishes")
parser.add_argument("--force", action="store_true", help="stop: force-stop instead of a graceful stop")
args = parser.parse_args()
api = Api()
if args.command == "inspect":
cmd_inspect(api)
elif args.command == "probe":
cmd_probe(api)
elif args.command == "existing":
cmd_existing(api)
elif args.command == "create":
cmd_create(api, args.keys, args.run)
elif args.command == "download":
cmd_download(api, args.keys)
elif args.command == "run-detail":
cmd_run_detail(api, args.keys)
elif args.command == "register-stage":
cmd_register_stage(api)
elif args.command == "stop":
cmd_stop(api, args.keys, args.force)
elif args.command == "artifacts":
cmd_artifacts(api, args.keys)
elif args.command == "worker":
cmd_worker(api)
elif args.command == "logs":
cmd_logs(api, args.keys)
elif args.command == "show":
cmd_show(api)
elif args.command == "status":
cmd_status(api, args.keys, args.wait)
elif args.command == "details":
cmd_details(api, args.keys)
elif args.command == "cat":
cmd_cat(api, args.keys)
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except RuntimeError as error:
print("ERROR:", error, file=sys.stderr)
sys.exit(1)