OpenPi_POC_Dev/ws_common/input_locate.py
2026-10-09 16:41:13 +08:00

158 lines
6.5 KiB
Python

"""上游条目定位与输入校验 (design_v5 §6.4 / §11 input_locate)。
读取 INPUT_ARTIFACTS_PATH 清单, 按 portId (或 name) 挑出本功能需要的条目,
数据本体在 path (容器内绝对路径, 直接读)。与 DATASET_PATH 超参二选一, 上游
优先; 两者都没有时非 0 退出。本镜像输入统一为挂载进容器的本地目录, 不做任何
网络存储下载: bos:// 等网络引用一律非 0 退出, 错误信息提示把数据集挂载进容器。
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
def _read_upstream_entries() -> list[dict]:
path = os.environ.get("INPUT_ARTIFACTS_PATH", "")
if not path or not os.path.isfile(path):
return []
try:
with open(path, "r", encoding="utf-8") as f:
entries = json.load(f)
except (json.JSONDecodeError, OSError) as exc:
sys.exit(f"[input] cannot read INPUT_ARTIFACTS_PATH={path}: {exc}")
return entries if isinstance(entries, list) else []
def _validate_local_dir(path: Path, label: str) -> Path:
if str(path).startswith("bos://") or "://" in str(path):
sys.exit(
f"[input] {label} path {path} is a network storage reference; this entry "
"accepts only container-local mounted directories. Mount the dataset into "
"the container and provide it via the upstream port or DATASET_PATH."
)
if not path.is_dir():
sys.exit(f"[input] {label} directory does not exist: {path}")
return path
def _read_json_list(env_key: str) -> list[dict]:
path = os.environ.get(env_key, "")
if not path or not os.path.isfile(path):
return []
try:
with open(path, "r", encoding="utf-8") as f:
items = json.load(f)
except (json.JSONDecodeError, OSError) as exc:
sys.exit(f"[input] cannot read {env_key}={path}: {exc}")
return items if isinstance(items, list) else []
def _binding_source_node(port_id: str) -> str:
"""平台契约: input-bindings.json 记录 targetPortId -> sourceNodeId 映射."""
for b in _read_json_list("INPUT_BINDINGS_PATH"):
if b.get("targetPortId") == port_id:
return str(b.get("sourceNodeId") or "").strip()
return ""
def _resolve_upstream_path(raw: str, entry: dict, port_id: str) -> str:
"""相对路径按平台约定解析: 上游产物 path 相对其产出节点 outputs 目录,
下游经 $WORKSPACE_ROOT/nodes/<sourceNodeId>/outputs/<path> 访问
(sourceNodeId 取 artifacts 条目 nodeId, 缺失时回退 input-bindings 的
sourceNodeId; 再兜底 $WORKSPACE_ROOT/nodes/*/outputs 下唯一匹配)。"""
if raw.startswith("/") or "://" in raw:
return raw
ws_root = os.environ.get("WORKSPACE_ROOT", "").strip()
if not ws_root:
return raw
node_id = str(entry.get("nodeId") or "").strip() or _binding_source_node(port_id)
if node_id:
return os.path.join(ws_root, "nodes", node_id, "outputs", raw)
import glob
hits = glob.glob(os.path.join(ws_root, "nodes", "*", "outputs", raw))
if len(hits) == 1:
return hits[0]
return raw
def is_lerobot_dataset_root(path: Path) -> bool:
"""LeRobot v3 数据集根的判据: meta/info.json 存在。"""
return (path / "meta" / "info.json").is_file()
def stage_to_local(root: Path) -> Path:
"""平台 workspace (网络 PVC, bosfs) 上的输入无条件暂存到节点本地盘。
训练随机读 PVC 实测慢 4-5 倍 (norm_stats ~29s/500 帧), 且多进程并发读网络盘
不稳定; 输入根位于 $WORKSPACE_ROOT 下时复制到 /tmp/pi-stage/<pid>/ (per-PID,
避免 torch DDP / jax 多 rank 竞态), 本地直跑场景 (不在 workspace 下) 原样返回。
与被否决的 PI_DATA_LOCAL 环境开关不同: 这是平台形态的内生行为, 不暴露超参。
"""
ws_root = os.environ.get("WORKSPACE_ROOT", "").strip()
if not ws_root:
return root
try:
root.resolve().relative_to(Path(ws_root).resolve())
except ValueError:
return root # 不在平台 workspace 下 (本地直跑), 不暂存
import shutil
dst_root = Path("/tmp/pi-stage") / str(os.getpid())
dst_root.mkdir(parents=True, exist_ok=True)
dst = dst_root / root.name
if dst.exists():
shutil.rmtree(dst)
print(f"[input] staging PVC input to node-local disk: {root} -> {dst}", flush=True)
shutil.copytree(root, dst)
print(f"[input] staging done ({dst})", flush=True)
return dst
def expand_dataset_roots(root: Path) -> list[Path]:
"""输入根分类 (多数据集聚合入口, 打破 D17 单数据集边界):
- 根自身是数据集 (有 meta/info.json) -> [根] (单数据集, 行为不变)
- 根是父目录 -> 其下每个含 meta/info.json 的子目录都是一个数据集,
按目录名排序返回 (确定性顺序); 一个都没有则非 0 退出
单端口承载多数据集 (平台一端口一上游约束下, 多数据集 = 一个父目录产物),
多数据集必须共父目录 (HF_LEROBOT_HOME 指向共同父目录, repo_id = 目录名)。
"""
if is_lerobot_dataset_root(root):
return [root]
subs = sorted(p for p in root.iterdir() if p.is_dir() and is_lerobot_dataset_root(p))
if not subs:
sys.exit(
f"[input] {root} is neither a LeRobot v3 dataset root (no meta/info.json) "
f"nor a parent of datasets; contents: {sorted(p.name for p in root.iterdir())[:10]}"
)
print(f"[input] multi-dataset root: {len(subs)} datasets under {root}: "
+ ", ".join(p.name for p in subs), flush=True)
return subs
def locate_input_dir(port_id: str = "dataset", env_key: str = "DATASET_PATH") -> Path:
"""上游端口 > 环境变量超参; 至少其一, 缺失非 0 退出。返回本地目录 Path。
返回的是输入根 (可能是单数据集根或多数据集父目录), 调用方用
expand_dataset_roots 展开成数据集列表。"""
for entry in _read_upstream_entries():
if entry.get("portId") == port_id or entry.get("name") == port_id:
raw = entry.get("path")
if not raw:
sys.exit(f"[input] upstream entry portId={port_id} has empty path")
return _validate_local_dir(Path(_resolve_upstream_path(raw, entry, port_id)), f"upstream port '{port_id}'")
value = os.environ.get(env_key, "").strip()
if value:
return _validate_local_dir(Path(value), env_key)
sys.exit(
f"[input] no input provided: neither upstream port '{port_id}' "
f"nor {env_key} is set. Provide at least one."
)