158 lines
6.5 KiB
Python
158 lines
6.5 KiB
Python
"""上游条目定位与输入校验 (design_v5 §6.4 / §11 input_locate)。
|
|
|
|
读取 INPUT_ARTIFACTS_PATH 清单, 按 portId (或 name) 挑出本功能需要的条目,
|
|
数据本体在 path (容器内绝对路径, 直接读)。与 DATASET_PATH 超参二选一, 上游
|
|
优先; 两者都没有时非 0 退出。本镜像输入统一为挂载进容器的本地目录, 不做任何
|
|
网络存储下载: bos:// 等网络引用一律非 0 退出, 错误信息提示把数据集挂载进容器。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def _read_upstream_entries() -> list[dict]:
|
|
path = os.environ.get("INPUT_ARTIFACTS_PATH", "")
|
|
if not path or not os.path.isfile(path):
|
|
return []
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
entries = json.load(f)
|
|
except (json.JSONDecodeError, OSError) as exc:
|
|
sys.exit(f"[input] cannot read INPUT_ARTIFACTS_PATH={path}: {exc}")
|
|
return entries if isinstance(entries, list) else []
|
|
|
|
|
|
def _validate_local_dir(path: Path, label: str) -> Path:
|
|
if str(path).startswith("bos://") or "://" in str(path):
|
|
sys.exit(
|
|
f"[input] {label} path {path} is a network storage reference; this entry "
|
|
"accepts only container-local mounted directories. Mount the dataset into "
|
|
"the container and provide it via the upstream port or DATASET_PATH."
|
|
)
|
|
if not path.is_dir():
|
|
sys.exit(f"[input] {label} directory does not exist: {path}")
|
|
return path
|
|
|
|
|
|
def _read_json_list(env_key: str) -> list[dict]:
|
|
path = os.environ.get(env_key, "")
|
|
if not path or not os.path.isfile(path):
|
|
return []
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
items = json.load(f)
|
|
except (json.JSONDecodeError, OSError) as exc:
|
|
sys.exit(f"[input] cannot read {env_key}={path}: {exc}")
|
|
return items if isinstance(items, list) else []
|
|
|
|
|
|
def _binding_source_node(port_id: str) -> str:
|
|
"""平台契约: input-bindings.json 记录 targetPortId -> sourceNodeId 映射."""
|
|
for b in _read_json_list("INPUT_BINDINGS_PATH"):
|
|
if b.get("targetPortId") == port_id:
|
|
return str(b.get("sourceNodeId") or "").strip()
|
|
return ""
|
|
|
|
|
|
def _resolve_upstream_path(raw: str, entry: dict, port_id: str) -> str:
|
|
"""相对路径按平台约定解析: 上游产物 path 相对其产出节点 outputs 目录,
|
|
下游经 $WORKSPACE_ROOT/nodes/<sourceNodeId>/outputs/<path> 访问
|
|
(sourceNodeId 取 artifacts 条目 nodeId, 缺失时回退 input-bindings 的
|
|
sourceNodeId; 再兜底 $WORKSPACE_ROOT/nodes/*/outputs 下唯一匹配)。"""
|
|
if raw.startswith("/") or "://" in raw:
|
|
return raw
|
|
ws_root = os.environ.get("WORKSPACE_ROOT", "").strip()
|
|
if not ws_root:
|
|
return raw
|
|
node_id = str(entry.get("nodeId") or "").strip() or _binding_source_node(port_id)
|
|
if node_id:
|
|
return os.path.join(ws_root, "nodes", node_id, "outputs", raw)
|
|
import glob
|
|
hits = glob.glob(os.path.join(ws_root, "nodes", "*", "outputs", raw))
|
|
if len(hits) == 1:
|
|
return hits[0]
|
|
return raw
|
|
|
|
|
|
def is_lerobot_dataset_root(path: Path) -> bool:
|
|
"""LeRobot v3 数据集根的判据: meta/info.json 存在。"""
|
|
return (path / "meta" / "info.json").is_file()
|
|
|
|
|
|
def stage_to_local(root: Path) -> Path:
|
|
"""平台 workspace (网络 PVC, bosfs) 上的输入无条件暂存到节点本地盘。
|
|
|
|
训练随机读 PVC 实测慢 4-5 倍 (norm_stats ~29s/500 帧), 且多进程并发读网络盘
|
|
不稳定; 输入根位于 $WORKSPACE_ROOT 下时复制到 /tmp/pi-stage/<pid>/ (per-PID,
|
|
避免 torch DDP / jax 多 rank 竞态), 本地直跑场景 (不在 workspace 下) 原样返回。
|
|
|
|
与被否决的 PI_DATA_LOCAL 环境开关不同: 这是平台形态的内生行为, 不暴露超参。
|
|
"""
|
|
ws_root = os.environ.get("WORKSPACE_ROOT", "").strip()
|
|
if not ws_root:
|
|
return root
|
|
try:
|
|
root.resolve().relative_to(Path(ws_root).resolve())
|
|
except ValueError:
|
|
return root # 不在平台 workspace 下 (本地直跑), 不暂存
|
|
import shutil
|
|
|
|
dst_root = Path("/tmp/pi-stage") / str(os.getpid())
|
|
dst_root.mkdir(parents=True, exist_ok=True)
|
|
dst = dst_root / root.name
|
|
if dst.exists():
|
|
shutil.rmtree(dst)
|
|
print(f"[input] staging PVC input to node-local disk: {root} -> {dst}", flush=True)
|
|
shutil.copytree(root, dst)
|
|
print(f"[input] staging done ({dst})", flush=True)
|
|
return dst
|
|
|
|
|
|
def expand_dataset_roots(root: Path) -> list[Path]:
|
|
"""输入根分类 (多数据集聚合入口, 打破 D17 单数据集边界):
|
|
|
|
- 根自身是数据集 (有 meta/info.json) -> [根] (单数据集, 行为不变)
|
|
- 根是父目录 -> 其下每个含 meta/info.json 的子目录都是一个数据集,
|
|
按目录名排序返回 (确定性顺序); 一个都没有则非 0 退出
|
|
|
|
单端口承载多数据集 (平台一端口一上游约束下, 多数据集 = 一个父目录产物),
|
|
多数据集必须共父目录 (HF_LEROBOT_HOME 指向共同父目录, repo_id = 目录名)。
|
|
"""
|
|
if is_lerobot_dataset_root(root):
|
|
return [root]
|
|
subs = sorted(p for p in root.iterdir() if p.is_dir() and is_lerobot_dataset_root(p))
|
|
if not subs:
|
|
sys.exit(
|
|
f"[input] {root} is neither a LeRobot v3 dataset root (no meta/info.json) "
|
|
f"nor a parent of datasets; contents: {sorted(p.name for p in root.iterdir())[:10]}"
|
|
)
|
|
print(f"[input] multi-dataset root: {len(subs)} datasets under {root}: "
|
|
+ ", ".join(p.name for p in subs), flush=True)
|
|
return subs
|
|
|
|
|
|
def locate_input_dir(port_id: str = "dataset", env_key: str = "DATASET_PATH") -> Path:
|
|
"""上游端口 > 环境变量超参; 至少其一, 缺失非 0 退出。返回本地目录 Path。
|
|
|
|
返回的是输入根 (可能是单数据集根或多数据集父目录), 调用方用
|
|
expand_dataset_roots 展开成数据集列表。"""
|
|
for entry in _read_upstream_entries():
|
|
if entry.get("portId") == port_id or entry.get("name") == port_id:
|
|
raw = entry.get("path")
|
|
if not raw:
|
|
sys.exit(f"[input] upstream entry portId={port_id} has empty path")
|
|
return _validate_local_dir(Path(_resolve_upstream_path(raw, entry, port_id)), f"upstream port '{port_id}'")
|
|
|
|
value = os.environ.get(env_key, "").strip()
|
|
if value:
|
|
return _validate_local_dir(Path(value), env_key)
|
|
|
|
sys.exit(
|
|
f"[input] no input provided: neither upstream port '{port_id}' "
|
|
f"nor {env_key} is set. Provide at least one."
|
|
)
|