first commit

This commit is contained in:
chao.wu 2026-10-09 16:41:13 +08:00
commit 6e8f7736fa
96 changed files with 12412 additions and 0 deletions

7
.dockerignore Normal file
View File

@ -0,0 +1,7 @@
.selfcheck/
.closedloop/
.local_dev/
*.log
__pycache__/
*.pyc
nohup.out

13
.gitignore vendored Normal file
View File

@ -0,0 +1,13 @@
# python 缓存
__pycache__/
*.pyc
.pytest_cache/
# 本地实验产物 (评测 dump / 训练日志 / jax 与 inductor 编译缓存)
.local_dev/
# 官方权重预取缓存 (12G, 由 scripts/fetch_weights.sh 三层来源获取: 本地 -> jfs -> GCS 在线)
weights_cache/
# openpi 上游源码 (克隆自 GitHub @15a9616 + 本地补丁, 由 scripts/fetch_openpi.sh 重建)
openpi/

70
Dockerfile Normal file
View File

@ -0,0 +1,70 @@
# OpenPi 训练/推理/转换镜像 train_inference_origin (design_v2 §1/§10)
# 构建: ./build_docker.sh (幂等预取官方权重到 weights_cache/, 镜像内只 COPY 不下载)
# 层序设计: apt -> 权重 COPY -> pip 依赖层(仅 requirements_train.txt 触发) -> 源码 COPY
# -> editable 安装层。改代码只重建最后两层, 40 分钟级 pip 依赖层保持缓存。
FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04
ARG http_proxy
ARG https_proxy
ARG no_proxy=localhost
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
python3.12 python3.12-venv python3.12-dev python3-pip \
build-essential git ca-certificates wget \
libgl1 libegl1 libglib2.0-0 libsm6 libxext6 libxrender1 \
libxkbcommon0 libjpeg8 zlib1g ffmpeg \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# 官方 JAX 权重 pi05_base + tokenizer (D8 只打这一份)。
COPY --chmod=a+rX weights_cache/openpi_weights/ /app/src/openpi_weights/
COPY requirements_train.txt /app/src/
# venv_train 依赖层: JAX + torch 双栈。只依赖 requirements_train.txt。
# 5090/5090D (sm_120) 需要 jax >= 0.7 (design_v2 D9 依赖可升级); torch 2.7.1+cu128
# 原生覆盖 sm_120。lerobot v3 (>=0.4.0) 与 data2lerobot 转换器同大版本线。
# 尾部同层 chmod: 避免末层 chmod -R /app 触发 overlay copy-up 双倍体积。
RUN python3.12 -m venv /app/src/venv_train \
&& /app/src/venv_train/bin/pip install --no-cache-dir torch==2.7.1 torchvision \
--index-url https://download.pytorch.org/whl/cu128 \
&& /app/src/venv_train/bin/pip install --no-cache-dir -r /app/src/requirements_train.txt \
&& /app/src/venv_train/bin/pip install --no-cache-dir --no-deps nvidia-nccl-cu12==2.29.7 \
&& chmod -R a+rX /app/src/venv_train
# 源码层
COPY --chmod=755 entry/ /app/entry/
COPY --chmod=755 report.sh /app/report.sh
COPY --chmod=a+rX scripts/ /app/src/scripts/
COPY --chmod=a+rX ws_common/ /app/src/ws_common/
COPY --chmod=a+rX pi_pipeline/ /app/src/pi_pipeline/
COPY --chmod=a+rX presets/ /app/src/presets/
COPY --chmod=a+rX tests/ /app/src/tests/
COPY --chmod=a+rX openpi/ /app/src/openpi
COPY --chmod=a+rX src-externals/XPolicyLab /app/src/src-externals/XPolicyLab
# editable 安装层: openpi 本体 (-e --no-deps) + transformers_replace 覆盖 + 自检。
RUN /app/src/venv_train/bin/pip install --no-cache-dir -e /app/src/openpi --no-deps \
&& /app/src/venv_train/bin/pip install --no-cache-dir -e /app/src/openpi/packages/openpi-client --no-deps \
&& SITE=$(/app/src/venv_train/bin/python -c "import transformers,os;print(os.path.dirname(transformers.__file__))") \
&& cp -r /app/src/openpi/src/openpi/models_pytorch/transformers_replace/* "$SITE/" \
&& /app/src/venv_train/bin/python -c \
"from transformers.models.siglip import check; assert check.check_whether_transformers_replace_is_installed_correctly()"
# 非 root (uid 10001) 可读可执行: 由 COPY --chmod 与各层内 chmod 保证, 无末层大 chmod。
# 运行时缓存目录: 平台容器以 uid 10001 运行, /app/src 不可写; 预置 tokenizer 到
# 缓存位 (big_vision) 避免运行时下载 gs://, 并放开缓存目录写权限。
RUN mkdir -p /app/src/openpi_cache/big_vision \
&& cp /app/src/openpi_weights/big_vision/paligemma_tokenizer.model /app/src/openpi_cache/big_vision/ \
&& chmod -R a+rwX /app/src/openpi_cache
ENV PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \
HOME=/tmp/pi-home XDG_CACHE_HOME=/tmp/pi-cache MPLCONFIGDIR=/tmp/pi-mpl \
TORCHINDUCTOR_CACHE_DIR=/tmp/pi-cache/inductor
# USER 固化为 pi: 平台容器以 uid 10001 运行且无 /etc/passwd 条目,
# torch.compile -> inductor -> getpass.getuser() 走 pwd.getpwuid 会崩; getpass
# 优先读 $USER 环境变量, 固定后免 passwd 条目 (原 containerCommand 注入转正)。
ENV USER=pi
ENV PYTHONPATH=/app/src/openpi/src:/app/src
# 无 ENTRYPOINT/CMD, 平台运行时覆盖

304
README.md Normal file
View File

@ -0,0 +1,304 @@
# OpenPi_Dev — Pi0.5 训练 / 转换 / 推理镜像工程
本项目将 openpi(Pi0.5, Physical Intelligence 开源实现)封装为一条数据流水线:输入 LeRobot v3 数据集,完成模型训练(JAX / PyTorch 双栈)、JAX 到 PyTorch 的权重转换、XPolicyLab 协议推理服务。
```
LeRobot v3 数据集 (RoboTwin2_Dev 产出或自备)
│
├─ train_jax / train_torch ──► checkpoint (含 model_structure.json 契约快照)
│ │
│ ├─ convert ──► torch 权重 (model.safetensors)
│ └─ serve_jax / serve_torch ──► XPolicyLab ws 协议服务
│ └─ RoboTwin 仿真评测客户端
└─ norm_stats (独立统计, 训练入口缺文件时也会自动补算)
```
全部功能由环境变量超参驱动,每个功能一个入口脚本,适配算子平台运行;也可以脱离平台直接 docker run 使用。
## 0. 当前已验证状态 (2026-09-08)
远端平台已通过简化套件 T1-T5 判定的发布物,与本地代码的对应关系:
| 项 | 值 |
|---|---|
| 镜像 tag | ccr-29eug8s3-pub.cnc.bj.baidubce.com/public/openpi_train_inference:da7ce8b-20260908-2 |
| openpi 副本 commit | da7ce8b (基于官方 15a9616, 见 openpi/ 内 git log) |
| 源码清单 | releases/manifest_da7ce8b-20260908-2.txt (构建上下文逐文件 sha256) |
| 平台算子 | JAX 训练 operator-9b489829810b7b59db63f30081fa0a07 / torch 训练 operator-848270666764d6ce59991129492ed7fa / norm_stats operator-d6ec15d9574813d2c47f3b716ce47dd6 (均 version 1.2.0, image 同上 tag) |
| 平台工作流 | workflow-cf61aefd254f0283327e1f9ec45e7dfa (project-3990926c0623c46d0f5942ff8febde23, OpenPi 平台测试项目) |
| 套件结果 | T1 jax 4cam / T2 torch 4cam / T3 norm_stats / T4 jax 1cam / T5 多数据集 4 库混合 全部 completed, run 依次 f36b9f5e / 4b7dcfe0 / 1ee7afb2 / 0bf63fb5 / 165fb507 |
| 套件驱动 | .local_dev/platform_test/platform_run.py + cases_t.json + cases_t5.json (复跑方法见 releases/RELEASES.md) |
本表之下的正文描述的是当前代码的功能语义(与该发布物一致)。重建该发布物:按 releases/RELEASES.md 的步骤构建并推送镜像即可,构建上下文即本目录当前状态。
## 1. 镜像
| 镜像 | 承载功能 | 说明 |
|---|---|---|
| openpi_train_inference | 训练 / 转换 / 推理 / 统计 | 基于 CUDA 12.8 + Python 3.12, 内置官方 JAX pi05_base 权重, 离线运行 |
构建(在本目录执行,tag 自动取 openpi 副本 git 短hash-日期并直推 CCR,同时写入 ~/tmp_daa_reg/cur_tag 供算子联动脚本读取):
```bash
./build_docker.sh # 训练/推理镜像 (default builder, --push)
```
训练镜像容器布局:入口在 /app/entry/,业务代码在 /app/src/ (pi_pipeline, ws_common, openpi 副本),官方权重固定在 /app/src/openpi_weights/pi05_base。运行时无 ENTRYPOINT,由调用方指定入口脚本:
```bash
docker run --rm --gpus all \
-e OUTPUT_DIR=/out \
-e DATASET_PATH=/data/my_lerobot_v3 \
-v $PWD/my_dataset:/data/my_lerobot_v3:ro \
-v $PWD/out:/out \
ccr-29eug8s3-pub.cnc.bj.baidubce.com/public/openpi_train_inference:<tag> \
sh /app/entry/train_jax.sh
```
## 2. 配置体系:模型参数与非模型参数
本项目的全部可配置项分为两类,理解这个划分是正确使用的前提:
- 模型参数:配置完成后,模型中每一个权重和激活对应的形状就已经确定。模型参数全部集中在契约 (CONTRACT_JSON) 里,训练 / 转换 / 推理全链路消费同一份契约,并被序列化进 checkpoint 的 model_structure.json 作为结构锚点。改模型参数意味着得到一个结构不同的模型,与既有 checkpoint 不兼容(action_dim 例外,官方权重 resume 时可自动 tile 适配)。
- 非模型参数:模型参数确定之后才生效的运行配置,包括训练过程、训练行为、恢复与产物管理。改非模型参数不影响任何权重形状,同类 checkpoint 之间可以自由更换。
### 2.1 模型参数(契约字段)
```json
{
"cameras": ["cam1", "cam2", "cam3", "cam4"],
"state": {"dim": 14, "inject": true},
"language": {"enabled": true, "max_token_len": 200},
"action": {"dim": 14, "horizon": 10},
"norm_mode": "min_max",
"key_map": {
"observation.images.cam_high": ["cam1"],
"observation.images.cam_third_view": ["cam2"],
"observation.images.cam_left_wrist": ["cam3"],
"observation.images.cam_right_wrist": ["cam4"],
"observation.state": "state",
"action": "actions",
"task": "prompt"
}
}
```
| 模型参数 | 决定的形状 |
|---|---|
| cameras | 相机槽位数量与拼接顺序, 决定视觉 token 总数与 prefix 结构; 槽名 cam1..camN, 理论上无上限 (受显存约束); key_map 中一个外部图像键可映射多个槽 (一对多复用) |
| 画面比例 | 固定 1:1: 224x224, 16x16 网格, 每路 256 token; 非 1:1 比例 (4:3/16:9 及 pos_emb 行切片) 已移除, 模型代码还原为官方状态, 旧契约出现 aspect_ratio 非 1:1 取值时非 0 退出 |
| state.dim 与 state.inject | 状态投影维度与是否注入 prefix; inject=false 时模型不接收状态输入 |
| action.dim | 动作维度, 决定 action_in_proj / action_out_proj 权重形状; 官方权重 action_dim=32, 与契约不一致时 resume 自动循环 tile 适配 (日志以 resume-adapt 标注) |
| action.horizon | 动作序列长度, 决定流匹配目标的时间维 |
| language.max_token_len | 语言指令 token 上限, 决定 prompt 激活宽度 |
| norm_mode | 归一化模式: min_max (默认, 有界到 -1..1)、mean_std、vision_only (等价 state.inject=false)、quantile (官方 pi05 语义); mean_std/min_max/quantile 互切不改形状, 仅数值分布不同 |
| LORA=true (环境变量) | 切换到低秩参数化, 权重集合与形状改变 (gemma_2b_lora + gemma_300m_lora), 冻结非 LoRA 参数并关闭 EMA |
key_map 校验失败(槽位冲突、槽位未映射、图像值非列表)会以非 0 退出并给出可读报错。不设 CONTRACT_JSON 时使用内置 RoboTwin 四相机默认契约(presets/robotwin_default_contract.json 等价)。
### 2.2 非模型参数(环境变量)
训练过程类(不改变模型结构,只影响训练如何进行):
| 参数 | 默认 | 说明 |
|---|---|---|
| BATCH_SIZE | 8 | 全局批大小; 多卡数据并行时须被数据并行度整除 |
| NUM_TRAIN_STEPS | 20000 | 训练步数 |
| LR_WARMUP_STEPS / LR_PEAK / LR_DECAY_STEPS | 1000 / 2.5e-5 / 30000 | 学习率日程 |
| EMA_DECAY | 0.99 | EMA 衰减系数; LORA=true 时自动关闭 |
| SEED | 0 | 随机种子 |
| NUM_WORKERS | 4 | 数据加载进程数 |
| FSDP_DEVICES | 1 | JAX 专属: 参数切分设备数, 1 为纯数据并行 |
| NPROC_PER_NODE | 1 | torch 专属: DDP 进程数, 大于 1 时入口自动以 torchrun 启动 |
训练行为类(冻结与精度,只动可训性与数值类型):
| 参数 | 说明 |
|---|---|
| FREEZE_BACKBONE | torch 专属: 冻结 Paligemma 主干, 只训 action_in_proj / action_out_proj / time_mlp 头, 低显存路径 |
| TRAIN_PRECISION | torch 专属: bfloat16 (默认) 或 float32 |
| OPENPI_DISABLE_AUGMENT | 置 1 关闭数据增广 (见注意事项) |
恢复与产物管理类:
| 参数 | 默认 | 说明 |
|---|---|---|
| RESUME | false | true 为断点续训, 从实验 checkpoint 目录的最新 step 恢复 |
| RESUME_CKPT | 空 (镜像内置官方权重) | JAX 专属: 外部权重目录, 指向 checkpoint 内的 params 目录 (如 .../99/params), 传 step 目录会报 item_metadata 错误 |
| PYTORCH_WEIGHT_PATH | 空 | torch 专属: torch 起点权重 (convert 产物), 空则随机初始化 (仅冒烟用) |
| NORM_STATS_PATH | 空 | 归一化统计文件; 空 = 自动统计 (存在即跳过, 幂等) |
| OUTPUT_DIR / EXP_NAME | 必填 / pi05_contract | 产物目录与实验名 |
| SAVE_INTERVAL / KEEP_PERIOD | 1000 / 5000 | 保存与长期保留节奏 |
| LOG_INTERVAL | 100 | 日志打印间隔 |
| OVERWRITE | false | true 时覆盖既有实验目录 |
## 3. 数据集要求
输入是标准 LeRobot Dataset v3.0 目录(含 meta/info.json、data/chunk-xxx/*.parquet、videos/),与 RoboTwin2_Dev 镜像的 data2lerobot 产物直接对接。数据格式差异会被自动处理,不需要预先转换:
- 视频帧 float32 (0,1) CHW 布局自动转 uint8 HWC(官方数据管线语义)
- 图像 letterbox 缩放对齐 PIL golden(含抗锯齿),四舍五入取整
- 多任务数据集(v3 tasks 索引)按帧的任务文本作为语言指令
数据集输入根支持两种形态(多数据集聚合):
- 根自身是数据集(有 meta/info.json):单数据集训练,行为与传统一致
- 根是父目录:其下每个含 meta/info.json 的子目录都是一个数据集,按目录名排序全部聚合训练(共享同一契约 transforms 与同一份混合 norm_stats,norm_stats 的资产目录名为 mix\<N\>-\<hash\>);多数据集必须共父目录(LeRobot 本地读取的 HF_LEROBOT_HOME 唯一),违例非 0 退出
平台运行时,位于 workspace PVC 下的输入根会被自动暂存到节点本地盘(/tmp/pi-stage/\<pid\>/)再训练——这是平台形态的内生行为,不暴露超参;本地直跑(输入不在 $WORKSPACE_ROOT 下)不暂存。
## 4. 功能一:JAX 训练 (train_jax.sh)
功能:契约驱动的 JAX 全参 / LoRA 微调,产出 checkpoint(权重 + train_state + norm_stats + model_structure.json 契约快照)。
配置:模型参数全部通过 CONTRACT_JSON 注入(即上文契约字段表);非模型参数全部环境变量注入,均可省略使用默认值。
镜像:openpi_train_inference。
```bash
docker run --rm --gpus '"device=0,1"' --shm-size=16g \
-e OUTPUT_DIR=/out -e DATASET_PATH=/data/my_lerobot_v3 \
-e CONTRACT_JSON='{"cameras":["cam1","cam2","cam3"],"state":{"dim":14,"inject":true},"language":{"enabled":true,"max_token_len":200},"action":{"dim":14,"horizon":10},"norm_mode":"min_max","key_map":{"observation.images.cam_high":["cam1"],"observation.images.cam_left_wrist":["cam2"],"observation.images.cam_right_wrist":["cam3"],"observation.state":"state","action":"actions","task":"prompt"}}' \
-e LORA=true -e NUM_TRAIN_STEPS=5000 \
-v $PWD/my_dataset:/data/my_lerobot_v3:ro -v $PWD/out:/out \
ccr-.../openpi_train_inference:<tag> sh /app/entry/train_jax.sh
```
多卡(JAX):可见 GPU 数须被 FSDP_DEVICES 整除,mesh 为 (数据并行度, 参数切分度),BATCH_SIZE 须被数据并行度整除:
| FSDP_DEVICES | 语义 | 适用 |
|---|---|---|
| 1 | 纯数据并行, 每卡完整模型, batch 按卡数均分 | 单卡装得下模型时扩展吞吐 |
| 2 / 4 | N 卡装一个模型 (参数沿特征维切分), 剩余卡数做数据并行 | 单卡装不下时降低单卡显存 |
已验证组合:2 卡 (FSDP=1/2)、4 卡 (FSDP=1/2/4)。
显存路线参考:
| 路线 | 参考占用 | 说明 |
|---|---|---|
| LORA=true (JAX) | 单卡约 15GB | 冻结主干转 bfloat16, 最省 |
| FREEZE_BACKBONE=1 (torch) | 单卡约 10GB (batch 2) | 只训练头部投影 |
| 全参微调 (JAX 多卡) | FSDP_DEVICES=2/4 分摊 | 实测 4x32GB (FSDP=4, BATCH_SIZE=1) 仍 OOM, 需 80GB 级卡; 32GB 卡建议 LoRA |
| 全参微调 (单卡) | 需 80GB 级显卡 | 官方默认配置 |
## 5. 功能二:PyTorch 训练 (train_torch.sh)
功能:与 JAX 入口同一契约、同一非模型参数面的 torch 训练;多卡为 DDP(每卡完整模型 + 梯度 all-reduce),由 NPROC_PER_NODE 大于 1 时自动 torchrun 启动。
配置:模型参数同契约;非模型参数在公共面之外多四个:PYTORCH_WEIGHT_PATH(起点权重)、TRAIN_PRECISION(bf16/fp32)、FREEZE_BACKBONE(冻结主干)、NPROC_PER_NODE(DDP 进程数)。已验证 DDP 组合:2 / 4 进程。
镜像:openpi_train_inference。
torch 训练建议从 convert 产物起步:先用 JAX 训练(或官方权重转换)得到 torch 权重,再设 PYTORCH_WEIGHT_PATH 续训;随机初始化仅用于链路冒烟。
## 6. 功能三:归一化统计 (norm_stats.sh)
功能:对数据集统计归一化参数,输出 norm_stats.json(含 mean/std/q01/q99/min/max),统计空间与训练管线一致(repack + 数据变换后)。训练入口发现缺失时也会自动补算;NORM_STATS_PATH 显式指定优先级最高。
配置:DATASET_PATH(必填)、CONTRACT_JSON、OUTPUT_DIR。无需 GPU。
镜像:openpi_train_inference。
## 7. 功能四:权重转换 (convert.sh)
功能:JAX checkpoint 转 torch model.safetensors,作为 torch 训练起点与 torch 推理的交接格式。附带能力:
- 结构判定读 checkpoint 内 model_structure.json(不依赖目录名);外部权重没有契约快照时用 CONTRACT_JSON 注入
- 转换后自动做 JAX 与 torch 前向数值对拍(余弦相似度阈值 0.95),结果写入 convert_metadata.json
- 产物附 paligemma_tokenizer.model
配置:CHECKPOINT_PATH(必填)、CONTRACT_JSON(外部权重时必填)、PRECISION(bfloat16 默认 / float32)、OUTPUT_DIR。GPU 可选。
镜像:openpi_train_inference。
转换 RoboTwin 官方 cotrain 权重(aloha_agilex)时,契约需要与官方训练配置对齐:3 相机(cam_high + 左右手腕,无第三视角)、action.dim=32、norm_mode=quantile。
## 8. 功能五:推理服务 (serve_jax.sh / serve_torch.sh)
功能:XPolicyLab websocket 协议动作服务(单客户端串行,动作 chunk 整块返回),与 RoboTwin2_Dev 镜像的 eval_client 直接联调。协议相机键经契约 key_map 逆映射到内部槽位;评测侧多出的相机自动忽略并记日志,契约要求但缺失的相机补零图并置 mask。
配置:
| 参数 | 默认 | 说明 |
|---|---|---|
| CHECKPOINT_PATH | 必填 | 契约管线 checkpoint 目录 (含 model_structure.json 与 assets/norm_stats) |
| CONTRACT_JSON | 空 | 外部权重无契约快照时注入 |
| SERVE_HOST / SERVE_PORT | 0.0.0.0 / 19000 | 监听地址 |
| DELTA_ACTION_SPACE | false | true = 模型输出为 delta joint (臂增量 + 爪绝对), 自动加回当前状态; 官方 aloha cotrain 权重必须开启 |
| TORCH_SERVE_DTYPE | bf16 | torch 服务精度; 设 fp32 时跳过 bfloat16 转换 |
| ACTION_TYPE | joint | joint 或 ee |
观测输入格式(XPolicyLab 协议):
```
obs["vision"][cam]["color"] uint8 RGB 图像 (H,W,3), cam 为对外键
(cam_head / cam_third_view / cam_left_wrist / cam_right_wrist)
obs["state"]["{left,right}_{arm,ee}_joint_state"] 关节状态, 拼接序 [左臂6,左爪1,右臂6,右爪1]
obs["instruction"] 语言指令
```
镜像:openpi_train_inference(单机单卡,docker run -p 暴露端口)。
与 RoboTwin 仿真评测客户端联调:
```bash
docker run --rm --gpus '"device=1"' \
-e NVIDIA_DRIVER_CAPABILITIES=all \
-e VK_ICD_FILENAMES=/etc/vulkan/icd.d/nvidia_icd.json \
-e OUTPUT_DIR=/out -e POLICY_SERVER_IP=<宿主IP> -e POLICY_SERVER_PORT=19000 \
-e TASK_NAME=beat_block_hammer -e TEST_NUM=5 -e SAVE_VIDEO=true \
-v $PWD/eval_out:/out \
ccr-29eug8s3-pub.cnc.bj.baidubce.com/public/robotwin2_data_debug:<tag> \
sh /app/entry/eval_client.sh
```
## 9. 产物结构
训练产物(两个训练入口同构):
```
$OUTPUT_DIR/
├── checkpoints/<exp>/<step>/ 模型参数 + train_state + assets/norm_stats.json
│ + model_structure.json (契约快照, serve/convert 的锚点)
├── train_report.json loss 摘要与契约快照
└── .artifacts.json 产物清单 (算子平台协议)
```
convert 产物:
```
$OUTPUT_DIR/checkpoint_torch/
├── model.safetensors torch 权重
├── model_structure.json 契约透传或注入契约生成
├── convert_metadata.json 数值对拍结果 (cosine / max_abs)
└── paligemma_tokenizer.model tokenizer 随行
```
## 10. 使用注意事项
1. torch 推理评测请使用 PRECISION=float32 的转换产物,并在服务侧设 TORCH_SERVE_DTYPE=fp32。开环数值对拍一致性很好的 bfloat16 权重,在闭环控制评测中成功率仍会明显下降。
2. RoboTwin 官方 cotrain 权重(RoboTwin-lerobot_v30-aloha_agilex)推理三要素:契约 3 相机(head + 双腕)、norm_mode=quantile、DELTA_ACTION_SPACE=true。缺 delta 还原的典型症状是机械臂几乎不动(模型输出 delta 增量被当作绝对关节角)。
3. 显存不足 32GB 的卡优先走 LoRA(JAX)或 FREEZE_BACKBONE(torch)路线;JAX 全参微调实测 4x32GB(FSDP=4、BATCH_SIZE=1)仍 OOM,需 80GB 级卡;torch LoRA 实测 32GB 单卡(含 BATCH_SIZE=1)与双卡 DDP 均超出显存,32GB 卡 torch 路线只有 FREEZE_BACKBONE 可用。
4. 多卡训练环境要求:NCCL 不低于 2.29.7(2.27.x 的 AllToAll 在 sm_120/虚拟化环境会报 unhandled system error),镜像内已内置;容器运行需足够共享内存(docker 默认 64MB 的 /dev/shm 会导致 NCCL 报 No space left on device,多卡训练须加 --shm-size=16g)。GPU 间跨 PCIe root(SYS 拓扑)时需设 NCCL_P2P_DISABLE=1(train_jax.sh 已内置默认值 1)。JAX 侧多卡支持 FSDP_DEVICES 数据并行与参数切分的任意整除组合,torch 侧为 DDP(每卡完整模型)。推理服务固定单机单卡。
5. 推理服务与转换只接受契约管线 checkpoint(目录内含 model_structure.json 与 assets/norm_stats.json)。官方原始权重需先经 convert(以 CONTRACT_JSON 注入契约)转成契约管线产物。
6. 容器内运行不需要外网:HF_HUB_OFFLINE 默认开启,tokenizer 与官方权重均已离线打入。
7. 训练数据增广(随机裁剪 / 旋转 / 色彩抖动)在 JAX 0.11 系存在第三方库兼容问题,冒烟可设 OPENPI_DISABLE_AUGMENT=1 关闭;该开关不影响推理。
8. 各入口遵循统一收尾协议:stdout 逐 step 打印 DROBOTICFLOW_PROGRESS 进度行,产物清单写 .artifacts.json,业务文件不占用 result.json 等平台保留文件名。
## 11. 目录结构(构建上下文)
```
OpenPi_Dev/
├── Dockerfile 训练/推理镜像 (CUDA 12.8 + Python 3.12)
├── build_docker.sh 训练镜像构建编排 (XPolicyLab 就位 / SDK wheels 暂存 / CCR 推送)
├── entry/ 七个功能入口脚本 (train / convert / serve / norm_stats)
├── pi_pipeline/ 业务包: contract / train_runner / train_torch_runner
│ serve_xpolicylab / convert_runner / norm_stats_runner
├── openpi/ openpi 副本 (@da7ce8b, 基于官方 15a9616), 深层改动唯一发生地, 文件头标注改动点
├── presets/ 默认契约模板
├── ws_common/ 与 RoboTwin2_Dev 同源的共享层 (输入定位 / 收尾协议)
├── releases/ 发布记录与构建清单 (RELEASES.md + 源码 sha256 manifest)
├── src-externals/XPolicyLab/ serve 协议实现 (构建期从 reference 拷入)
└── tests/ 契约校验 / letterbox golden 一致性单测
```

42
build_docker.sh Executable file
View File

@ -0,0 +1,42 @@
#!/bin/bash
# 幂等构建编排 (train 镜像)。
# 用法: ./build_docker.sh (无需参数)
set -euo pipefail
cd "$(dirname "$0")"
# 1) XPolicyLab 就位 (serve 前端要 import 其 client_server/ws 服务端协议实现)
DEST="src-externals/XPolicyLab"
if [ ! -d "$DEST/client_server" ]; then
mkdir -p "$DEST"
echo "[build] copying ../reference/XPolicyLab -> $DEST"
(cd "../reference/XPolicyLab" && tar cf - --exclude='.git' .) | (cd "$DEST" && tar xf -)
fi
# 2) 官方权重幂等预取 (Dockerfile 只 COPY, 镜像内不下载)
PROXY="${http_proxy:-${https_proxy:-http://192.168.16.68:18000}}"
export http_proxy="$PROXY" https_proxy="$PROXY"
bash scripts/fetch_weights.sh
# 3) buildx builder: 默认 docker driver (宿主 daemon 的 systemd 代理 + registry 凭证,
# 拉取/推送都可靠)。容器型 builder 的 registry 请求不走 env 代理且创建时固化的
# 代理地址会过期, 不要用 (2026-09-08 实测, 详见 releases/RELEASES.md)。
PROXY="${http_proxy:-${https_proxy:-http://192.168.16.68:18000}}"
export http_proxy="$PROXY" https_proxy="$PROXY"
# 4) tag = openpi 副本 git 短hash-日期 (镜像身份自带代码可追溯, 对齐平台官方算子规范);
# 同日多次构建手动加 -N 后缀区分。tag 同时落 ~/tmp_daa_reg/cur_tag 供
# update_ops.py (算子联动) 与 platform_run.py (IMG_TRAIN) 读取。
GIT_HASH="$(git -C openpi rev-parse --short HEAD)"
STAMP="$(date +%Y%m%d)"
REG="ccr-29eug8s3-pub.cnc.bj.baidubce.com/public"
image="$REG/openpi_train_inference:${GIT_HASH}-${STAMP}"
mkdir -p "$HOME/tmp_daa_reg"
echo -n "$image" > "$HOME/tmp_daa_reg/cur_tag"
# --push 直推 registry, 不 --load 落本地 (省一半磁盘)
docker buildx build --builder default --push \
--build-arg http_proxy="$PROXY" \
--build-arg https_proxy="$PROXY" \
--build-arg no_proxy="localhost" \
-f Dockerfile -t "$image" .
echo "[build] pushed: $image"

25
entry/convert.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/sh
# JAX -> torch 权重转换入口 (design_v2 §8)。
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
echo "== container env self-check (log only, failures tolerated) =="
df -h || true
free -h || true
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
: "${CHECKPOINT_PATH:?CHECKPOINT_PATH not set (or connect upstream checkpoint port)}"
# 幂等清理: 只清本功能产物
rm -rf "$OUTPUT_DIR/checkpoint_torch"
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
export OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/app/src/openpi_cache}"
/app/src/venv_train/bin/python /app/src/pi_pipeline/convert_runner.py
sh /app/report.sh

36
entry/norm_stats.sh Executable file
View File

@ -0,0 +1,36 @@
#!/bin/sh
# norm_stats 独立统计入口 (design_v2 §6, D11)。输入 dataset 端口或 DATASET_PATH,
# 输出 norm_stats 产物端口 (norm_stats.json, 含 mean/std/q01/q99/min/max)。
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
echo "== container env self-check (log only, failures tolerated) =="
df -h || true
free -h || true
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
# stdout/stderr 双写: 控制台 + $OUTPUT_DIR/stdout.log (PVC)。
# 平台对失败节点的容器 stdout 不持久化 (Job GC 即失), 落盘才能事后取证。
LOG_PIPE="$OUTPUT_DIR/.stdout.pipe"
rm -f "$LOG_PIPE"
if mkfifo "$LOG_PIPE" 2>/dev/null; then
tee "$OUTPUT_DIR/stdout.log" < "$LOG_PIPE" &
TEE_PID=$!
exec > "$LOG_PIPE" 2>&1
else
exec > "$OUTPUT_DIR/stdout.log" 2>&1
fi
# 幂等清理: 只清本功能产物
rm -rf "$OUTPUT_DIR/norm_stats"
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
export HF_LEROBOT_HOME="${HF_LEROBOT_HOME:-$OUTPUT_DIR}"
/app/src/venv_train/bin/python /app/src/pi_pipeline/norm_stats_runner.py
sh /app/report.sh

26
entry/serve_jax.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/sh
# Pi05 JAX 推理服务 (XPolicyLab ws 协议前端, design_v2 §7)。
# 不注册算子 (D7): docker run -p 映射端口对外, 常驻服务不走收尾协议。
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
echo "== container env self-check (log only, failures tolerated) =="
df -h || true
free -h || true
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
: "${CHECKPOINT_PATH:?CHECKPOINT_PATH not set (or connect upstream checkpoint port)}"
SERVE_HOST="${SERVE_HOST:-0.0.0.0}"
SERVE_PORT="${SERVE_PORT:-19000}"
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
export OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/app/src/openpi_cache}"
export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}"
export XLA_PYTHON_CLIENT_PREALLOCATE="${XLA_PYTHON_CLIENT_PREALLOCATE:-false}"
# 常驻服务: exec 替换为业务进程 (无收尾), 平台/运维按需终止
exec /app/src/venv_train/bin/python /app/src/pi_pipeline/serve_xpolicylab.py

25
entry/serve_torch.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/sh
# Pi05 torch 推理服务 (XPolicyLab ws 协议前端, design_v2 §7)。
# 权重由 checkpoint 目录内 model.safetensors 自动判定 (j2t 产物或 torch 训练产物)。
# 不注册算子 (D7): docker run -p 映射端口对外, 常驻服务不走收尾协议。
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
echo "== container env self-check (log only, failures tolerated) =="
df -h || true
free -h || true
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
: "${CHECKPOINT_PATH:?CHECKPOINT_PATH not set (or connect upstream checkpoint port)}"
SERVE_HOST="${SERVE_HOST:-0.0.0.0}"
SERVE_PORT="${SERVE_PORT:-19000}"
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
export OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/app/src/openpi_cache}"
export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}"
exec /app/src/venv_train/bin/python /app/src/pi_pipeline/serve_xpolicylab.py

55
entry/train_jax.sh Executable file
View File

@ -0,0 +1,55 @@
#!/bin/sh
# Pi05 JAX 训练入口 (design_v2 §5)。契约 CONTRACT_JSON 整体注入 (默认 RoboTwin 四相机)。
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
if ! (
set -eu
echo "== container env self-check (log only, failures tolerated) =="
df -h || true
free -h || true
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
# stdout/stderr 双写: 控制台 + $OUTPUT_DIR/stdout.log (PVC)。
# 平台对失败节点的容器 stdout 不持久化 (Job GC 即失), 落盘才能事后取证。
LOG_PIPE="$OUTPUT_DIR/.stdout.pipe"
rm -f "$LOG_PIPE"
if mkfifo "$LOG_PIPE" 2>/dev/null; then
tee "$OUTPUT_DIR/stdout.log" < "$LOG_PIPE" &
TEE_PID=$!
exec > "$LOG_PIPE" 2>&1
else
exec > "$OUTPUT_DIR/stdout.log" 2>&1
fi
EXP_NAME="${EXP_NAME:-pi05_contract}"
# 幂等清理: 只清本功能产物 (checkpoints/assets/train_report), 不动平台协作文件。
# RESUME=true 语义为断点续训 (checkpoint manager 从既有 step 目录恢复), 与此处
# 清理不冲突: 续训场景由使用方保证不重复触发清理 (同节点重跑 = 全新训练)。
rm -rf "$OUTPUT_DIR/checkpoints" "$OUTPUT_DIR/assets" "$OUTPUT_DIR/train_report.json"
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
export OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/app/src/openpi_cache}"
export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}"
export XLA_PYTHON_CLIENT_PREALLOCATE="${XLA_PYTHON_CLIENT_PREALLOCATE:-false}"
# 多卡: GPU 间跨 PCIe root (SYS 拓扑) 时 PCIe P2P 不可用, NCCL 报 unhandled
# system error; 统一禁 P2P 走 host memory (PHB 拓扑下仅轻微降效)
export NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-1}"
export PI_CKPT_LOCAL="${PI_CKPT_LOCAL:-}"
/app/src/venv_train/bin/python /app/src/pi_pipeline/train_runner.py
); then
echo "[entry] train_jax FAILED; env keys: $(env | cut -d= -f1 | sort | tr '\n' ' ')"
echo "[entry] keeping pod alive 900s for diagnosis"
sleep 900
exit 1
fi
sh /app/report.sh

55
entry/train_torch.sh Executable file
View File

@ -0,0 +1,55 @@
#!/bin/sh
# Pi05 torch 训练入口 (design_v2 §5.3)。torch 起点权重经 PYTORCH_WEIGHT_PATH
# (j2t 产物或用户外部权重); 单机多卡经 NPROC_PER_NODE (torchrun, D14)。
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
if ! (
set -eu
echo "== container env self-check (log only, failures tolerated) =="
df -h || true
free -h || true
nvidia-smi || echo "[env-check] nvidia-smi unavailable, continue without GPU info"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
# stdout/stderr 双写: 控制台 + $OUTPUT_DIR/stdout.log (PVC)。
# 平台对失败节点的容器 stdout 不持久化 (Job GC 即失), 落盘才能事后取证。
LOG_PIPE="$OUTPUT_DIR/.stdout.pipe"
rm -f "$LOG_PIPE"
if mkfifo "$LOG_PIPE" 2>/dev/null; then
tee "$OUTPUT_DIR/stdout.log" < "$LOG_PIPE" &
TEE_PID=$!
exec > "$LOG_PIPE" 2>&1
else
exec > "$OUTPUT_DIR/stdout.log" 2>&1
fi
# 幂等清理: 只清本功能产物
rm -rf "$OUTPUT_DIR/checkpoints" "$OUTPUT_DIR/assets" "$OUTPUT_DIR/train_report.json"
export PYTHONPATH="/app/src/openpi/src:/app/src${PYTHONPATH:+:$PYTHONPATH}"
export OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/app/src/openpi_cache}"
export HF_HUB_OFFLINE="${HF_HUB_OFFLINE:-1}"
NPROC="${NPROC_PER_NODE:-1}"
if [ "$NPROC" -gt 1 ]; then
# torchrun 走 venv 全路径 (镜像无系统级 torchrun, 不依赖 PATH)
/app/src/venv_train/bin/torchrun --standalone --nnodes=1 --nproc_per_node="$NPROC" \
/app/src/pi_pipeline/train_torch_runner.py
else
/app/src/venv_train/bin/python /app/src/pi_pipeline/train_torch_runner.py
fi
); then
echo "[entry] train_torch FAILED; env keys: $(env | cut -d= -f1 | sort | tr '\n' ' ')"
echo "[entry] keeping pod alive 900s for diagnosis"
sleep 900
exit 1
fi
sh /app/report.sh

View File

@ -0,0 +1,925 @@
From da7ce8b6720afdbd28054a3ca9d804eb60c9f9f8 Mon Sep 17 00:00:00 2001
From: "chao01.wu" <chao01.wu@local>
Date: Tue, 8 Sep 2026 21:28:01 +0800
Subject: [PATCH] DataPipeline: 1:1-only revert (aspect/pos_emb slice removed,
siglip+modeling_siglip restored to official) + multi-dataset concat
(DataConfig.extra_repo_ids, create_torch_dataset Concat)
---
examples/convert_jax_model_to_pytorch.py | 6 +-
scripts/train_pytorch.py | 19 ++-
src/openpi/models/gemma.py | 13 +-
src/openpi/models/model.py | 40 ++++++-
src/openpi/models/pi0.py | 40 ++++++-
src/openpi/models/pi0_config.py | 25 ++--
src/openpi/models_pytorch/pi0_pytorch.py | 10 +-
.../models_pytorch/preprocessing_pytorch.py | 5 +
src/openpi/shared/array_typing.py | 22 ++--
src/openpi/shared/image_tools.py | 10 +-
src/openpi/shared/nnx_utils.py | 8 +-
src/openpi/shared/normalize.py | 7 +-
src/openpi/training/checkpoints.py | 8 +-
src/openpi/training/config.py | 8 ++
src/openpi/training/data_loader.py | 59 +++++++--
src/openpi/training/sharding.py | 75 ++++++++++--
src/openpi/training/utils.py | 6 +
src/openpi/transforms.py | 112 +++++++++++++++---
18 files changed, 402 insertions(+), 71 deletions(-)
diff --git a/examples/convert_jax_model_to_pytorch.py b/examples/convert_jax_model_to_pytorch.py
index 632c0b8..e4459bb 100644
--- a/examples/convert_jax_model_to_pytorch.py
+++ b/examples/convert_jax_model_to_pytorch.py
@@ -290,7 +290,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
llm_mlp_linear = state_dict.pop(f"llm/layers/mlp_{num_expert}/linear{suffix}")
# Check if we have Dense layers (for pi05/adaptive normalization) or scale layers (for regular pi0)
- if "pi05" in checkpoint_dir:
+ if pi05:
# Pi05 with adaptive normalization
llm_input_layernorm_bias = state_dict.pop(f"llm/layers/pre_attention_norm_{num_expert}/Dense_0/bias{suffix}")
llm_post_attention_layernorm_bias = state_dict.pop(f"llm/layers/pre_ffw_norm_{num_expert}/Dense_0/bias{suffix}")
@@ -345,7 +345,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
i
].transpose()
- if "pi05" in checkpoint_dir:
+ if pi05:
# Pi05 with adaptive normalization - use Dense layer parameters directly
state_dict[f"paligemma_with_expert.gemma_expert.model.layers.{i}.input_layernorm.dense.bias"] = (
llm_input_layernorm_bias[i]
@@ -369,7 +369,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
)
# Handle final norm layer
- if "pi05" in checkpoint_dir:
+ if pi05:
# Pi05 with adaptive normalization - use Dense layer parameters directly
final_norm_bias = state_dict.pop(f"llm/final_norm_{num_expert}/Dense_0/bias{suffix}")
final_norm_kernel = state_dict.pop(f"llm/final_norm_{num_expert}/Dense_0/kernel{suffix}")
diff --git a/scripts/train_pytorch.py b/scripts/train_pytorch.py
index c7ddd2b..fa698ca 100644
--- a/scripts/train_pytorch.py
+++ b/scripts/train_pytorch.py
@@ -455,8 +455,25 @@ def train_loop(config: _config.TrainConfig):
end_lr = config.lr_schedule.decay_lr
# Create optimizer with config parameters
+ # DataPipeline 改动 (design_v2 §5.3 注记): FREEZE_BACKBONE=1 冻结 Paligemma 主干
+ # (vision/llm/expert), 只训练 action_in_proj/action_out_proj/time_mlp 头 —
+ # 32GB 卡全参 AdamW (params+grads+opt 状态 ~36GB fp32) 不可行时的低显存路径
+ # (官方 torch 栈无 LoRA 支持, 冒烟/验收语义为可训练可转化, 不要求全参收敛)
+ trainable_params = model.parameters()
+ if os.environ.get("FREEZE_BACKBONE", "").strip().lower() in ("1", "true", "yes"):
+ frozen, trainable = 0, 0
+ for name, p in model.named_parameters():
+ if any(head in name for head in ("action_in_proj", "action_out_proj", "time_mlp")):
+ p.requires_grad = True
+ trainable += p.numel()
+ else:
+ p.requires_grad = False
+ frozen += p.numel()
+ trainable_params = [p for p in model.parameters() if p.requires_grad]
+ logging.info(f"[freeze] trainable={trainable/1e6:.1f}M frozen={frozen/1e6:.1f}M params")
+
optim = torch.optim.AdamW(
- model.parameters(),
+ trainable_params,
lr=peak_lr,
betas=(config.optimizer.b1, config.optimizer.b2),
eps=config.optimizer.eps,
diff --git a/src/openpi/models/gemma.py b/src/openpi/models/gemma.py
index d1623d1..7fbb113 100644
--- a/src/openpi/models/gemma.py
+++ b/src/openpi/models/gemma.py
@@ -146,7 +146,18 @@ class Embedder(nn.Module):
)
def encode(self, x):
- x = self.input_embedding_table[(x,)]
+ # DataPipeline 改动: jax>=0.11 下词表 gather (replicated embedding 表 x
+ # batch 分片 indices) 无法自动解析输出 sharding (模型内 prompt 与 state
+ # 离散 token concat 后继承 batch 分片)。按 jax 官方姿势显式声明输出
+ # 分片 = indices 各维 + embedding 维, 维度 0 对齐 batch 轴。
+ mesh = sharding.active_mesh()
+ if mesh is not None and mesh.devices.size > 1:
+ spec = jax.sharding.PartitionSpec(sharding.BATCH_AXIS, *((None,) * (x.ndim - 1)))
+ x = self.input_embedding_table.at[x].get(
+ out_sharding=jax.sharding.NamedSharding(mesh, spec)
+ )
+ else:
+ x = self.input_embedding_table[(x,)]
x *= jnp.sqrt(self.embed_dim).astype(x.dtype)
return x
diff --git a/src/openpi/models/model.py b/src/openpi/models/model.py
index 29618b4..073e802 100644
--- a/src/openpi/models/model.py
+++ b/src/openpi/models/model.py
@@ -3,6 +3,7 @@ from collections.abc import Sequence
import dataclasses
import enum
import logging
+import os
import pathlib
from typing import Generic, TypeVar
@@ -81,6 +82,10 @@ IMAGE_RESOLUTION = (224, 224)
@at.typecheck
@struct.dataclass
class Observation(Generic[ArrayT]):
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) 同 TrainState — __no_type_check__ 跳过 jaxtyped dataclass __init__
+ # 检查 (jaxtyping>=0.11 tracing 下 images dict 以 None 占位初始化被误拒)
+ __no_type_check__ = True
"""Holds observations, i.e., inputs to the model.
See `Observation.from_dict` to see the expected dictionary form. This is the format
@@ -141,6 +146,20 @@ class Observation(Generic[ArrayT]):
Actions = at.Float[ArrayT, "*b ah ad"]
+def _reshard_to_data_axis(x):
+ """DataPipeline 改动: jax>=0.11 vmap/concat 的 sharding 一致性辅助。
+
+ 多进程 mesh 下把数组 reshard 到数据轴 (与 training.sharding.data_axis_spec
+ 同语义: fsdp=1 时退化单 batch 轴); 无 mesh 上下文 (单进程推理/转换自检) 时
+ 原样返回。
+ """
+ mesh = jax.sharding.get_abstract_mesh()
+ if not mesh.empty and mesh.size > 1:
+ axis = ("batch",) if mesh.shape["fsdp"] == 1 else ("batch", "fsdp")
+ x = jax.sharding.reshard(x, jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(axis)))
+ return x
+
+
def preprocess_observation(
rng: at.KeyArrayLike | None,
observation: Observation,
@@ -165,7 +184,11 @@ def preprocess_observation(
logger.info(f"Resizing image {key} from {image.shape[1:3]} to {image_resolution}")
image = image_tools.resize_with_pad(image, *image_resolution)
- if train:
+ if train and os.environ.get("OPENPI_DISABLE_AUGMENT", "").strip().lower() not in ("1", "true", "yes"):
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) OPENPI_DISABLE_AUGMENT=1 跳过 augmax 增广 — augmax (jax 0.4
+ # 时代库) 与 jax>=0.11 不兼容 (ColorJitter pixelwise 内部崩, 风险项
+ # R-13), 冒烟/推理不受影响; 正式训练前需替换或修复增广实现
# Convert from [-1, 1] to [0, 1] for augmax.
image = image / 2.0 + 0.5
@@ -181,6 +204,12 @@ def preprocess_observation(
augmax.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5),
]
sub_rngs = jax.random.split(rng, image.shape[0])
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) jax>=0.11 要求 vmap 消去维度的 sharding 一致且全 Explicit mesh
+ # 下 with_sharding_constraint 是 assert — 用 reshard 把 split 产生的
+ # replicated sub_rngs 真分片到数据轴 (与 image 一致); 无 mesh 上下文
+ # (单进程推理/转换自检) 时跳过
+ sub_rngs = _reshard_to_data_axis(sub_rngs)
image = jax.vmap(augmax.Chain(*transforms))(sub_rngs, image)
# Back to [-1, 1].
@@ -312,7 +341,14 @@ def restore_params(
with ocp.PyTreeCheckpointer() as ckptr:
metadata = ckptr.metadata(params_path)
- item = {"params": metadata["params"]}
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) orbax>=0.12 的 metadata() 返回 StepMetadata (树结构在
+ # item_metadata["params"]), 旧版直接下标; 双形态兼容
+ if hasattr(metadata, "item_metadata"):
+ params_metadata = metadata.item_metadata["params"]
+ else:
+ params_metadata = metadata["params"]
+ item = {"params": params_metadata}
params = ckptr.restore(
params_path,
diff --git a/src/openpi/models/pi0.py b/src/openpi/models/pi0.py
index ae7c459..09dddda 100644
--- a/src/openpi/models/pi0.py
+++ b/src/openpi/models/pi0.py
@@ -11,6 +11,7 @@ from openpi.models import model as _model
from openpi.models import pi0_config
import openpi.models.gemma as _gemma
import openpi.models.siglip as _siglip
+import openpi.training.sharding as _sharding
from openpi.shared import array_typing as at
logger = logging.getLogger("openpi")
@@ -67,6 +68,9 @@ class Pi0(_model.BaseModel):
def __init__(self, config: pi0_config.Pi0Config, rngs: nnx.Rngs):
super().__init__(config.action_dim, config.action_horizon, config.max_token_len)
self.pi05 = config.pi05
+ # DataPipeline 改动 (D15/D2): 保存 config 引用, preprocess_observation 与
+ # SigLIP 构造从契约派生字段取图像槽位/分辨率/pos_emb 行切片。
+ self.config = config
paligemma_config = _gemma.get_config(config.paligemma_variant)
action_expert_config = _gemma.get_config(config.action_expert_variant)
# TODO: rewrite gemma in NNX. For now, use bridge.
@@ -177,7 +181,12 @@ class Pi0(_model.BaseModel):
action_expert_tokens = action_time_tokens
adarms_cond = None
tokens.append(action_expert_tokens)
- input_mask.append(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
+ # 来源: openpi仓库 src/openpi/models/pi0.py @ 15a9616
+ # 改动点: 1) jax>=0.11 要求 concat 操作数 sharding 一致 — jit 内 jnp.ones 生成的
+ # suffix_mask 是 replicated, 与来自数据分片的 prefix_mask 冲突; 约束到数据轴
+ # (无 mesh 上下文时跳过)
+ suffix_mask = _model._reshard_to_data_axis(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
+ input_mask.append(suffix_mask)
# image/language/state inputs do not attend to action tokens
ar_mask += [True] + ([False] * (self.action_horizon - 1))
tokens = jnp.concatenate(tokens, axis=1)
@@ -190,9 +199,20 @@ class Pi0(_model.BaseModel):
self, rng: at.KeyArrayLike, observation: _model.Observation, actions: _model.Actions, *, train: bool = False
) -> at.Float[at.Array, "*b ah"]:
preprocess_rng, noise_rng, time_rng = jax.random.split(rng, 3)
- observation = _model.preprocess_observation(preprocess_rng, observation, train=train)
+ # DataPipeline 改动 (D15): 图像槽位/分辨率取契约派生字段 (默认官方值)。
+ observation = _model.preprocess_observation(
+ preprocess_rng,
+ observation,
+ train=train,
+ image_keys=self.config.resolved_image_keys,
+ )
batch_shape = actions.shape[:-2]
+ # DataPipeline 改动: jax>=0.11 下不同子路径 (图像/语言 vs actions/time)
+ # 的自然分片可能物理等价但 spec 不同 (batch 轴份额 1 时 P('batch') 与
+ # P(('batch','fsdp')) 并存), concatenate 严格比较即报错; 入口统一
+ # reshard 到维度 0 对齐 batch 轴
+ observation = _sharding.activation_sharding_constraint(observation)
noise = jax.random.normal(noise_rng, actions.shape)
time = jax.random.beta(time_rng, 1.5, 1, batch_shape) * 0.999 + 0.001
time_expanded = time[..., None, None]
@@ -202,6 +222,14 @@ class Pi0(_model.BaseModel):
# one big forward pass of prefix + suffix at once
prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation)
suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(observation, x_t, time)
+ # DataPipeline 改动: prefix (observation 路径) 与 suffix (actions/time 路径)
+ # 的自然分片在 batch 轴份额 1 时会得到 P('batch') 与 P(('batch','fsdp'))
+ # 两种物理等价但 spec 不同的布局, concatenate 严格比较报错; 统一 reshard
+ _unify = _sharding.activation_sharding_constraint
+ prefix_tokens, prefix_mask, prefix_ar_mask = _unify((prefix_tokens, prefix_mask, prefix_ar_mask))
+ suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = _unify(
+ (suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond)
+ )
input_mask = jnp.concatenate([prefix_mask, suffix_mask], axis=1)
ar_mask = jnp.concatenate([prefix_ar_mask, suffix_ar_mask], axis=0)
attn_mask = make_attn_mask(input_mask, ar_mask)
@@ -222,7 +250,13 @@ class Pi0(_model.BaseModel):
num_steps: int | at.Int[at.Array, ""] = 10,
noise: at.Float[at.Array, "b ah ad"] | None = None,
) -> _model.Actions:
- observation = _model.preprocess_observation(None, observation, train=False)
+ # DataPipeline 改动 (D15): 同 compute_loss, 契约派生槽位/分辨率。
+ observation = _model.preprocess_observation(
+ None,
+ observation,
+ train=False,
+ image_keys=self.config.resolved_image_keys,
+ )
# note that we use the convention more common in diffusion literature, where t=1 is noise and t=0 is the target
# distribution. yes, this is the opposite of the pi0 paper, and I'm sorry.
dt = -1.0 / num_steps
diff --git a/src/openpi/models/pi0_config.py b/src/openpi/models/pi0_config.py
index 584d83f..078c628 100644
--- a/src/openpi/models/pi0_config.py
+++ b/src/openpi/models/pi0_config.py
@@ -32,6 +32,11 @@ class Pi0Config(_model.BaseModelConfig):
# This config option is not used directly by the model, but it is read by the ModelTransformFactory.
discrete_state_input: bool = None # type: ignore
+ # DataPipeline 改动 (design_v2 D15, 契约驱动): 相机槽位。
+ # 默认 None = 官方行为 (model.IMAGE_KEYS 3 槽, 224x224, 完整 16x16 网格), 向后兼容。
+ # 契约注入示例: image_keys=("cam1","cam2","cam3","cam4")
+ image_keys: tuple[str, ...] | None = None
+
pytorch_compile_mode: str | None = "max-autotune"
def __post_init__(self):
@@ -47,6 +52,11 @@ class Pi0Config(_model.BaseModelConfig):
"max-autotune-no-cudagraphs",
]
+ @property
+ def resolved_image_keys(self) -> tuple[str, ...]:
+ """契约槽位或官方默认 (cam 序号即 prefix token 拼接顺序, D15)。"""
+ return self.image_keys if self.image_keys else _model.IMAGE_KEYS
+
@property
@override
def model_type(self) -> _model.ModelType:
@@ -62,21 +72,16 @@ class Pi0Config(_model.BaseModelConfig):
@override
def inputs_spec(self, *, batch_size: int = 1) -> tuple[_model.Observation, _model.Actions]:
+ # DataPipeline 改动 (D15): 图像槽位由契约派生 (resolved_image_keys),
+ # 默认与官方一致 (3 槽 224x224)。
image_spec = jax.ShapeDtypeStruct([batch_size, *_model.IMAGE_RESOLUTION, 3], jnp.float32)
image_mask_spec = jax.ShapeDtypeStruct([batch_size], jnp.bool_)
+ keys = self.resolved_image_keys
with at.disable_typechecking():
observation_spec = _model.Observation(
- images={
- "base_0_rgb": image_spec,
- "left_wrist_0_rgb": image_spec,
- "right_wrist_0_rgb": image_spec,
- },
- image_masks={
- "base_0_rgb": image_mask_spec,
- "left_wrist_0_rgb": image_mask_spec,
- "right_wrist_0_rgb": image_mask_spec,
- },
+ images={key: image_spec for key in keys},
+ image_masks={key: image_mask_spec for key in keys},
state=jax.ShapeDtypeStruct([batch_size, self.action_dim], jnp.float32),
tokenized_prompt=jax.ShapeDtypeStruct([batch_size, self.max_token_len], jnp.int32),
tokenized_prompt_mask=jax.ShapeDtypeStruct([batch_size, self.max_token_len], bool),
diff --git a/src/openpi/models_pytorch/pi0_pytorch.py b/src/openpi/models_pytorch/pi0_pytorch.py
index e68ddb7..f8dee9d 100644
--- a/src/openpi/models_pytorch/pi0_pytorch.py
+++ b/src/openpi/models_pytorch/pi0_pytorch.py
@@ -160,8 +160,14 @@ class PI0Pytorch(nn.Module):
return torch.where(att_2d_masks_4d, 0.0, -2.3819763e38)
def _preprocess_observation(self, observation, *, train=True):
- """Helper method to preprocess observation."""
- observation = _preprocessing.preprocess_observation_pytorch(observation, train=train)
+ """Helper model to preprocess observation."""
+ # DataPipeline 改动 (design_v2 D15): 图像槽位取契约派生字段
+ # (Pi0Config.resolved_image_keys, 默认官方值), 与 JAX 侧一致。
+ observation = _preprocessing.preprocess_observation_pytorch(
+ observation,
+ train=train,
+ image_keys=self.config.resolved_image_keys,
+ )
return (
list(observation.images.values()),
list(observation.image_masks.values()),
diff --git a/src/openpi/models_pytorch/preprocessing_pytorch.py b/src/openpi/models_pytorch/preprocessing_pytorch.py
index 33c94a5..d53cbc7 100644
--- a/src/openpi/models_pytorch/preprocessing_pytorch.py
+++ b/src/openpi/models_pytorch/preprocessing_pytorch.py
@@ -39,6 +39,11 @@ def preprocess_observation_pytorch(
# TODO: This is a hack to handle both [B, C, H, W] and [B, H, W, C] formats
# Handle both [B, C, H, W] and [B, H, W, C] formats
+ # DataPipeline 改动: HWC 输入 (B,H,W,C: 尾维==3 且 shape[1]!=3) 先归一到
+ # CHW — 官方 torch 数据管线为 CHW, 但 LeRobot 侧共用 transform 链为 HWC;
+ # 下游 embed_image -> transformers get_image_features 只接受 CHW
+ if image.shape[-1] == 3 and image.shape[1] != 3:
+ image = image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
is_channels_first = image.shape[1] == 3 # Check if channels are in dimension 1
if is_channels_first:
diff --git a/src/openpi/shared/array_typing.py b/src/openpi/shared/array_typing.py
index 569eafe..0be9e74 100644
--- a/src/openpi/shared/array_typing.py
+++ b/src/openpi/shared/array_typing.py
@@ -26,21 +26,25 @@ import torch
# the problem is that custom PyTree nodes are sometimes initialized with arbitrary types (e.g., `jax.ShapeDtypeStruct`,
# `jax.Sharding`, or even <object>) due to JAX tracing operations. this patch skips typechecking when the stack trace
# contains `jax._src.tree_util`, which should only be the case during tree unflattening.
-_original_check_dataclass_annotations = jaxtyping._decorator._check_dataclass_annotations # noqa: SLF001
+# 来源: openpi仓库 src/openpi/shared/array_typing.py @ 15a9616
+# 改动点: 1) jaxtyping>=0.3 移除了 _check_dataclass_annotations 私有 API (issue #277 已上游修复),
+# 改为存在性判断: 旧版打 patch, 新版跳过
+_original_check_dataclass_annotations = getattr(jaxtyping._decorator, "_check_dataclass_annotations", None) # noqa: SLF001
# Redefine Array to include both JAX arrays and PyTorch tensors
Array = jax.Array | torch.Tensor
-def _check_dataclass_annotations(self, typechecker):
- if not any(
- frame.frame.f_globals.get("__name__") in {"jax._src.tree_util", "flax.nnx.transforms.compilation"}
- for frame in inspect.stack()
- ):
- return _original_check_dataclass_annotations(self, typechecker)
- return None
+if _original_check_dataclass_annotations is not None:
+ def _check_dataclass_annotations(self, typechecker):
+ if not any(
+ frame.frame.f_globals.get("__name__") in {"jax._src.tree_util", "flax.nnx.transforms.compilation"}
+ for frame in inspect.stack()
+ ):
+ return _original_check_dataclass_annotations(self, typechecker)
+ return None
-jaxtyping._decorator._check_dataclass_annotations = _check_dataclass_annotations # noqa: SLF001
+ jaxtyping._decorator._check_dataclass_annotations = _check_dataclass_annotations # noqa: SLF001
KeyArrayLike: TypeAlias = jax.typing.ArrayLike
Params: TypeAlias = PyTree[Float[ArrayLike, "..."]]
diff --git a/src/openpi/shared/image_tools.py b/src/openpi/shared/image_tools.py
index 8cde353..6c5c034 100644
--- a/src/openpi/shared/image_tools.py
+++ b/src/openpi/shared/image_tools.py
@@ -90,8 +90,16 @@ def resize_with_pad_torch(
resized_width = int(cur_width / ratio)
# Resize
+ # 来源: openpi仓库 src/openpi/shared/image_tools.py @ 15a9616
+ # 改动点: 1) bilinear 时开 antialias=True, 对齐 golden (PIL BILINEAR 缩小时为抗锯齿
+ # scale-aware 三角核; jax.image.resize 默认 antialias=True 天然一致,
+ # F.interpolate 默认 False 导致缩小场景与 golden 大范围偏差)
resized_images = F.interpolate(
- images, size=(resized_height, resized_width), mode=mode, align_corners=False if mode == "bilinear" else None
+ images,
+ size=(resized_height, resized_width),
+ mode=mode,
+ align_corners=False if mode == "bilinear" else None,
+ antialias=True if mode == "bilinear" else False,
)
# Handle dtype-specific clipping
diff --git a/src/openpi/shared/nnx_utils.py b/src/openpi/shared/nnx_utils.py
index 29df222..5ea9bf9 100644
--- a/src/openpi/shared/nnx_utils.py
+++ b/src/openpi/shared/nnx_utils.py
@@ -65,5 +65,9 @@ class PathRegex:
def state_map(state: nnx.State, filter: nnx.filterlib.Filter, fn: Callable[[Any], Any]) -> nnx.State:
"""Apply a function to the leaves of the state that match the filter."""
- filtered_keys = set(state.filter(filter).flat_state())
- return state.map(lambda k, v: fn(v) if k in filtered_keys else v)
+ # 来源: openpi仓库 src/openpi/shared/nnx_utils.py @ 15a9616
+ # 改动点: 1) flax>=0.12 的 State.flat_state() 返回 FlatState 序列 (旧版 dict),
+ # set(...flat_state()) 对 Param 不可 hash 报错; 改为 to_predicate 逐叶判断,
+ # 两版语义一致
+ predicate = nnx.filterlib.to_predicate(filter)
+ return state.map(lambda k, v: fn(v) if predicate(tuple(k), v) else v)
diff --git a/src/openpi/shared/normalize.py b/src/openpi/shared/normalize.py
index 5a75fae..f56daea 100644
--- a/src/openpi/shared/normalize.py
+++ b/src/openpi/shared/normalize.py
@@ -12,6 +12,10 @@ class NormStats:
std: numpydantic.NDArray
q01: numpydantic.NDArray | None = None # 1st quantile
q99: numpydantic.NDArray | None = None # 99th quantile
+ # DataPipeline 改动 (design_v2 D4): 真 min/max 统计, 供 min_max 归一化模式。
+ # 旧 norm_stats.json 缺字段时为 None, 消费方回退直方图分位数并 INFO。
+ min: numpydantic.NDArray | None = None
+ max: numpydantic.NDArray | None = None
class RunningStats:
@@ -83,7 +87,8 @@ class RunningStats:
variance = self._mean_of_squares - self._mean**2
stddev = np.sqrt(np.maximum(0, variance))
q01, q99 = self._compute_quantiles([0.01, 0.99])
- return NormStats(mean=self._mean, std=stddev, q01=q01, q99=q99)
+ # DataPipeline 改动 (D4): 暴露真 min/max (RunningStats 内部本就维护)。
+ return NormStats(mean=self._mean, std=stddev, q01=q01, q99=q99, min=self._min, max=self._max)
def _adjust_histograms(self):
"""Adjust histograms when min or max changes."""
diff --git a/src/openpi/training/checkpoints.py b/src/openpi/training/checkpoints.py
index f32a831..fbdd572 100644
--- a/src/openpi/training/checkpoints.py
+++ b/src/openpi/training/checkpoints.py
@@ -41,8 +41,12 @@ def initialize_checkpoint_dir(
checkpoint_dir,
item_handlers={
"assets": CallbackHandler(),
- "train_state": ocp.PyTreeCheckpointHandler(),
- "params": ocp.PyTreeCheckpointHandler(),
+ # use_ocdbt=False: OCDBT 的 manifest 原子写语义在 NFS 类网络盘
+ # (平台 workspace PVC) 上会产生读回不一致 (DATA_LOSS: Length in
+ # header does not match); 关闭后走树状文件布局, 网络盘安全。
+ # 读取侧 orbax 自动识别格式, 官方 OCDBT 权重恢复不受影响。
+ "train_state": ocp.PyTreeCheckpointHandler(use_ocdbt=False),
+ "params": ocp.PyTreeCheckpointHandler(use_ocdbt=False),
},
options=ocp.CheckpointManagerOptions(
max_to_keep=1,
diff --git a/src/openpi/training/config.py b/src/openpi/training/config.py
index 4ca47e1..6b3e2d8 100644
--- a/src/openpi/training/config.py
+++ b/src/openpi/training/config.py
@@ -81,6 +81,14 @@ class DataConfig:
model_transforms: _transforms.Group = dataclasses.field(default_factory=_transforms.Group)
# If true, will use quantile normalization. Otherwise, normal z-score normalization will be used.
use_quantile_norm: bool = False
+ # DataPipeline 改动 (design_v2 D4): 契约归一化模式显式注入 (None -> 官方
+ # use_quantile_norm 行为): "min_max" / "mean_std" / "quantile"。
+ norm_mode: str | None = None
+
+ # DataPipeline 改动 (design_v2 多数据集聚合): 主 repo_id 之外的附加数据集
+ # repo_id 列表 (共父目录, 与主数据集共用同一套 transforms / norm_stats)。
+ # create_torch_dataset 会对主 + 附加逐一建库并 ConcatDataset 拼接。
+ extra_repo_ids: tuple[str, ...] = ()
# Names of keys that will be used by the data loader to generate the action sequence. The length of the
# sequence is defined by the `action_horizon` field in the model config. This should be adjusted if your
diff --git a/src/openpi/training/data_loader.py b/src/openpi/training/data_loader.py
index e2ee7dd..9f11269 100644
--- a/src/openpi/training/data_loader.py
+++ b/src/openpi/training/data_loader.py
@@ -1,4 +1,5 @@
from collections.abc import Iterator, Sequence
+import dataclasses
import logging
import multiprocessing
import os
@@ -7,7 +8,12 @@ from typing import Literal, Protocol, SupportsIndex, TypeVar
import jax
import jax.numpy as jnp
-import lerobot.common.datasets.lerobot_dataset as lerobot_dataset
+# DataPipeline 改动 (design_v2 D1): LeRobot v3 读取 (lerobot.datasets.* 命名空间,
+# 与 data2lerobot 转换器同大版本线 >=0.4.0); v2 安装时回退旧命名空间。
+try:
+ import lerobot.datasets.lerobot_dataset as lerobot_dataset
+except ImportError: # pragma: no cover - v2 fallback
+ import lerobot.common.datasets.lerobot_dataset as lerobot_dataset
import numpy as np
import torch
@@ -127,6 +133,28 @@ class FakeDataset(Dataset):
return self._num_samples
+def _create_single_lerobot_dataset(repo_id: str, data_config: _config.DataConfig, action_horizon: int):
+ """DataPipeline 改动 (多数据集聚合): 单 repo 的 LeRobot 数据集构建, 供主/附加
+ 数据集复用; delta_timestamps 的秒换算按各数据集自身 fps (fps 可不同)。"""
+ dataset_meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
+ dataset = lerobot_dataset.LeRobotDataset(
+ repo_id,
+ delta_timestamps={
+ key: [t / dataset_meta.fps for t in range(action_horizon)] for key in data_config.action_sequence_keys
+ },
+ )
+
+ if data_config.prompt_from_task:
+ # 改动点: lerobot >=0.4 的 meta.tasks 为 DataFrame(index=任务文本, 列含 task_index),
+ # PromptFromLeRobotTask 期望 dict[int, str] (0.1.0 语义), 此处归一化。
+ tasks = dataset_meta.tasks
+ if hasattr(tasks, "loc"): # pandas DataFrame
+ tasks = {int(row.task_index): idx for idx, row in tasks.iterrows()}
+ dataset = TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(tasks)])
+
+ return dataset
+
+
def create_torch_dataset(
data_config: _config.DataConfig, action_horizon: int, model_config: _model.BaseModelConfig
) -> Dataset:
@@ -137,16 +165,19 @@ def create_torch_dataset(
if repo_id == "fake":
return FakeDataset(model_config, num_samples=1024)
- dataset_meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
- dataset = lerobot_dataset.LeRobotDataset(
- data_config.repo_id,
- delta_timestamps={
- key: [t / dataset_meta.fps for t in range(action_horizon)] for key in data_config.action_sequence_keys
- },
- )
+ dataset = _create_single_lerobot_dataset(repo_id, data_config, action_horizon)
- if data_config.prompt_from_task:
- dataset = TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(dataset_meta.tasks)])
+ # DataPipeline 改动 (多数据集聚合): 附加数据集与主数据集共用同一 DataConfig 的
+ # transforms / norm_stats (repo_id 逐个替换), ConcatDataset 顺序拼接。
+ extra_repo_ids = list(getattr(data_config, "extra_repo_ids", ()) or ())
+ if extra_repo_ids:
+ parts = [dataset]
+ for extra_id in extra_repo_ids:
+ sub_config = dataclasses.replace(data_config, repo_id=extra_id, extra_repo_ids=())
+ parts.append(_create_single_lerobot_dataset(extra_id, sub_config, action_horizon))
+ dataset = torch.utils.data.ConcatDataset(parts)
+ logging.info(f"multi-dataset concat: {len(parts)} datasets "
+ f"({', '.join([repo_id, *extra_repo_ids])}), total {len(dataset)} frames")
return dataset
@@ -180,12 +211,18 @@ def transform_dataset(dataset: Dataset, data_config: _config.DataConfig, *, skip
)
norm_stats = data_config.norm_stats
+ # DataPipeline 改动 (D4): norm_mode 显式注入 (None -> 官方 use_quantile_norm 行为)。
return TransformedDataset(
dataset,
[
*data_config.repack_transforms.inputs,
*data_config.data_transforms.inputs,
- _transforms.Normalize(norm_stats, use_quantiles=data_config.use_quantile_norm),
+ _transforms.Normalize(
+ norm_stats,
+ use_quantiles=data_config.use_quantile_norm,
+ mode=data_config.norm_mode,
+ clip_to_quantiles=data_config.norm_mode == "mean_std",
+ ),
*data_config.model_transforms.inputs,
],
)
diff --git a/src/openpi/training/sharding.py b/src/openpi/training/sharding.py
index 344c550..ec4f382 100644
--- a/src/openpi/training/sharding.py
+++ b/src/openpi/training/sharding.py
@@ -7,13 +7,37 @@ import numpy as np
BATCH_AXIS = "batch"
FSDP_AXIS = "fsdp"
# In FSDP, we shard the data across both the batch and FSDP axes.
+# DataPipeline 注记: jax>=0.11 下当参数维恰可被 fsdp 整除时 (如 6 卡的 1152),
+# 数据复合轴与参数 fsdp 切分混合的 contraction 会报非法 sharding (旧版 jax 隐式
+# all-gather 不再自动插入); LoRA/低显存路径用 FSDP_DEVICES=1 (数据并行, 参数
+# replicate) 回避, 全量 FSDP 路径需显式 reshard 改造 (待 80GB 卡场景处理)
DATA_AXIS = (BATCH_AXIS, FSDP_AXIS)
+def data_axis_spec(mesh: jax.sharding.Mesh) -> tuple[str, ...]:
+ """DataPipeline 改动: 数据分片轴统一为纯 batch 轴。
+
+ - 单设备: 空 spec (replicated) — jax>=0.11 Explicit mesh 下任何轴名标记
+ (即使份额为 1) 都会触发词表 embedding gather 的输出分片歧义断言;
+ - 多设备: 仅 'batch' 轴 (数据并行度 = device_count / fsdp_devices)。
+ 官方复合轴 (batch, fsdp) 在 jax>=0.11 下与参数 fsdp 切分混合的
+ contraction 不再自动插入 all-gather (DenseGeneral 输出 fsdp 双重
+ 占用 → 非法 sharding), 故数据不再沾 fsdp 轴; fsdp 轴仅用于参数。
+ """
+ if mesh.devices.size == 1:
+ return ()
+ return (BATCH_AXIS,)
+
+
class _MeshState:
active_mesh: jax.sharding.Mesh | None = None
+def active_mesh() -> jax.sharding.Mesh | None:
+ """DataPipeline 改动: 模型层 (gemma 词表 gather 的显式 out_sharding) 需要只读访问当前 mesh。"""
+ return _MeshState.active_mesh
+
+
def make_mesh(num_fsdp_devices: int) -> jax.sharding.Mesh:
if jax.device_count() % num_fsdp_devices != 0:
raise ValueError(
@@ -38,11 +62,26 @@ def set_mesh(mesh: jax.sharding.Mesh):
def activation_sharding_constraint(pytree):
- if _MeshState.active_mesh is None:
+ # DataPipeline 改动: jax>=0.11 Explicit mesh 下 with_sharding_constraint 是
+ # 断言 (静态 spec 无法匹配 contraction 的自然分片); 改用 jax.sharding.reshard
+ # 强制层边界激活布局 = 维度 0 对齐 batch 轴、特征维 replicated (层间 all-gather,
+ # 即 FSDP 的标准层边界语义; MLP 中间大激活保持 fsdp 切分省显存)。
+ # 这同时满足 scan 对 carry 进出同型的要求。
+ if _MeshState.active_mesh is None or _MeshState.active_mesh.shape[FSDP_AXIS] == 1:
return pytree
- return jax.lax.with_sharding_constraint(
- pytree, jax.sharding.NamedSharding(_MeshState.active_mesh, jax.sharding.PartitionSpec(DATA_AXIS))
- )
+ mesh = _MeshState.active_mesh
+
+ def _reshard(a):
+ if not hasattr(a, "shape") or not hasattr(a, "dtype"):
+ return a
+ # 1 维数组 (如 per-token 的 ar_mask [N]) 无 batch 维, 标记 dim0 为 batch
+ # 会在 broadcast 后污染目标维并阻塞 reduce; 只处理 dim0 为 batch 的数组
+ if len(a.shape) < 2:
+ return a
+ spec = jax.sharding.PartitionSpec(*((BATCH_AXIS,) + (None,) * (len(a.shape) - 1)))
+ return jax.sharding.reshard(a, jax.sharding.NamedSharding(mesh, spec))
+
+ return jax.tree_util.tree_map(_reshard, pytree)
def fsdp_sharding(
@@ -80,15 +119,35 @@ def fsdp_sharding(
if (arr_size := np.prod(array.shape) * np.dtype(array.dtype).itemsize) < min_size_bytes:
return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
- # shard matrices and larger tensors along the largest axis that is divisible by the fsdp dimension
- axes = np.argsort(array.shape)[::-1]
- spec = [None] * len(axes)
- for i in axes:
+ # DataPipeline 改动: attention 类参数一律 replicate。scan 将 27/18 层堆叠
+ # 为单个大权重, 阈值无法按层粒度放行; 而 attention 权重任何维度的切分
+ # 都会进入收缩维 (qkv 的 head_dim 在 MHA einsum 收缩; out proj 的输入
+ # 维在 attn_vec_einsum 收缩), jax>=0.11 下双边 fsdp 收缩必须显式
+ # out_sharding (flax MHA 内部不可注入), 故整体 replicate (约 1GB,
+ # 相对总参数是小头)。
+ _kp = jax.tree_util.keystr(kp)
+ if (
+ "MultiHeadDotProductAttention" in _kp
+ or "['attn']" in _kp
+ or "attn_vec_einsum" in _kp
+ or "q_einsum" in _kp
+ or "kv_einsum" in _kp
+ ):
+ return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
+
+ # DataPipeline 改动: 轴选择从"最大可整除轴优先"改为"最后维优先"。
+ # flax Dense kernel 为 [in, out], 收缩维是 dim0; 切最后维 (out 维) 时
+ # contraction 输出可自然分片, 不产生双边 fsdp 收缩歧义 (jax>=0.11 不再
+ # 自动插入 all-gather)。词表 embedding [vocab, d] 切 d 维后, gemma.encode
+ # 的 gather 以显式 out_sharding (d 维 replicated) 收敛, 即 FSDP 的
+ # 用时 all-gather 语义。
+ for i in reversed(range(len(array.shape))):
if array.shape[i] % mesh.shape[FSDP_AXIS] == 0:
if log:
logging.info(
f"Sharding {jax.tree_util.keystr(kp)} of shape {array.shape} ({arr_size / 2**20:.2f} MiB) along axis {i}"
)
+ spec = [None] * len(array.shape)
spec[i] = FSDP_AXIS
return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(*spec))
diff --git a/src/openpi/training/utils.py b/src/openpi/training/utils.py
index fe7f94d..bc7d155 100644
--- a/src/openpi/training/utils.py
+++ b/src/openpi/training/utils.py
@@ -13,6 +13,12 @@ from openpi.shared import array_typing as at
@at.typecheck
@struct.dataclass
class TrainState:
+ # 来源: openpi仓库 src/openpi/training/utils.py @ 15a9616
+ # 改动点: 1) 设 __no_type_check__ 跳过 jaxtyped 对 dataclass __init__ 的运行时检查 —
+ # jaxtyping>=0.3 移除了 _check_dataclass_annotations 豁免挂点 (issue #277 上游重构),
+ # eval_shape/tracing 下 step 会被以 ShapeDtypeStruct / NamedSharding 等抽象占位
+ # 初始化, beartype 误拒; __no_type_check__ 是 jaxtyped 对 dataclass 的官方跳过开关
+ __no_type_check__ = True
step: at.Int[at.ArrayLike, ""]
params: nnx.State
model_def: nnx.GraphDef[_model.BaseModel]
diff --git a/src/openpi/transforms.py b/src/openpi/transforms.py
index 272375e..8764739 100644
--- a/src/openpi/transforms.py
+++ b/src/openpi/transforms.py
@@ -118,25 +118,43 @@ class Normalize(DataTransformFn):
use_quantiles: bool = False
# If true, will raise an error if any of the keys in the norm stats are not present in the data.
strict: bool = False
+ # DataPipeline 改动 (design_v2 D4/D19): 归一化模式显式化。
+ # None -> 由 use_quantiles 推断 (官方行为: quantile / mean_std);
+ # "min_max": 有界 -> [-1,1] (min_max 统计, 兼容 discrete_state_input, 契约默认);
+ # "quantile" / "mean_std": 与官方两档一致。
+ mode: str | None = None
+ # mean_std + discrete_state 注入时 clip 到 q01/q99 的归一化像 (D19, 对齐作者
+ # quantile 有界化意图), 避免 256-bin 离散化截断。
+ clip_to_quantiles: bool = False
def __post_init__(self):
- if self.norm_stats is not None and self.use_quantiles:
+ resolved_mode = self.mode or ("quantile" if self.use_quantiles else "mean_std")
+ if resolved_mode not in ("mean_std", "quantile", "min_max"):
+ raise ValueError(f"unsupported normalize mode: {resolved_mode}")
+ if self.norm_stats is not None and resolved_mode == "quantile":
_assert_quantile_stats(self.norm_stats)
+ object.__setattr__(self, "mode", resolved_mode)
def __call__(self, data: DataDict) -> DataDict:
if self.norm_stats is None:
return data
- return apply_tree(
- data,
- self.norm_stats,
- self._normalize_quantile if self.use_quantiles else self._normalize,
- strict=self.strict,
- )
+ fn = {
+ "mean_std": self._normalize,
+ "quantile": self._normalize_quantile,
+ "min_max": self._normalize_min_max,
+ }[self.mode]
+ return apply_tree(data, self.norm_stats, fn, strict=self.strict)
def _normalize(self, x, stats: NormStats):
mean, std = stats.mean[..., : x.shape[-1]], stats.std[..., : x.shape[-1]]
- return (x - mean) / (std + 1e-6)
+ out = (x - mean) / (std + 1e-6)
+ if self.clip_to_quantiles and stats.q01 is not None and stats.q99 is not None:
+ q01, q99 = stats.q01[..., : x.shape[-1]], stats.q99[..., : x.shape[-1]]
+ lo = (q01 - mean) / (std + 1e-6)
+ hi = (q99 - mean) / (std + 1e-6)
+ out = np.clip(out, lo, hi)
+ return out
def _normalize_quantile(self, x, stats: NormStats):
assert stats.q01 is not None
@@ -144,28 +162,44 @@ class Normalize(DataTransformFn):
q01, q99 = stats.q01[..., : x.shape[-1]], stats.q99[..., : x.shape[-1]]
return (x - q01) / (q99 - q01 + 1e-6) * 2.0 - 1.0
+ def _normalize_min_max(self, x, stats: NormStats):
+ lo = stats.min if stats.min is not None else stats.q01
+ hi = stats.max if stats.max is not None else stats.q99
+ assert lo is not None and hi is not None, (
+ "min_max mode requires min/max stats (recompute norm_stats); "
+ "legacy stats fallback to q01/q99"
+ )
+ lo, hi = lo[..., : x.shape[-1]], hi[..., : x.shape[-1]]
+ return (x - lo) / (hi - lo + 1e-6) * 2.0 - 1.0
+
@dataclasses.dataclass(frozen=True)
class Unnormalize(DataTransformFn):
norm_stats: at.PyTree[NormStats] | None
# If true, will use quantile normalization. Otherwise, normal z-score normalization will be used.
use_quantiles: bool = False
+ # DataPipeline 改动 (D4): 与 Normalize.mode 对称。
+ mode: str | None = None
def __post_init__(self):
- if self.norm_stats is not None and self.use_quantiles:
+ resolved_mode = self.mode or ("quantile" if self.use_quantiles else "mean_std")
+ if resolved_mode not in ("mean_std", "quantile", "min_max"):
+ raise ValueError(f"unsupported unnormalize mode: {resolved_mode}")
+ if self.norm_stats is not None and resolved_mode == "quantile":
_assert_quantile_stats(self.norm_stats)
+ object.__setattr__(self, "mode", resolved_mode)
def __call__(self, data: DataDict) -> DataDict:
if self.norm_stats is None:
return data
+ fn = {
+ "mean_std": self._unnormalize,
+ "quantile": self._unnormalize_quantile,
+ "min_max": self._unnormalize_min_max,
+ }[self.mode]
# Make sure that all the keys in the norm stats are present in the data.
- return apply_tree(
- data,
- self.norm_stats,
- self._unnormalize_quantile if self.use_quantiles else self._unnormalize,
- strict=True,
- )
+ return apply_tree(data, self.norm_stats, fn, strict=True)
def _unnormalize(self, x, stats: NormStats):
mean = pad_to_dim(stats.mean, x.shape[-1], axis=-1, value=0.0)
@@ -180,6 +214,16 @@ class Unnormalize(DataTransformFn):
return np.concatenate([(x[..., :dim] + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01, x[..., dim:]], axis=-1)
return (x + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01
+ def _unnormalize_min_max(self, x, stats: NormStats):
+ lo = stats.min if stats.min is not None else stats.q01
+ hi = stats.max if stats.max is not None else stats.q99
+ assert lo is not None and hi is not None
+ if (dim := lo.shape[-1]) < x.shape[-1]:
+ return np.concatenate(
+ [(x[..., :dim] + 1.0) / 2.0 * (hi - lo + 1e-6) + lo, x[..., dim:]], axis=-1
+ )
+ return (x + 1.0) / 2.0 * (hi - lo + 1e-6) + lo
+
@dataclasses.dataclass(frozen=True)
class ResizeImages(DataTransformFn):
@@ -324,6 +368,44 @@ class PromptFromLeRobotTask(DataTransformFn):
return {**data, "prompt": prompt}
+@dataclasses.dataclass(frozen=True)
+class ImagesToHWC(DataTransformFn):
+ """DataPipeline 改动 (design_v2 D1): 对齐官方 libero_policy._parse_image 语义 —
+ lerobot >=0.4 视频帧为 float32 [0,1] CHW, 官方数据管线为 uint8 HWC
+ (ResizeImages 走 PIL uint8 golden; Observation.from_dict 再转 [-1,1] float,
+ torch 分支并 permute CHW)。float 输入 clip [0,1] 后 *255 取整, 对
+ uint8/HWC 输入幂等无操作。
+ """
+
+ def __call__(self, data: DataDict) -> DataDict:
+ images = data.get("image")
+ if not isinstance(images, dict):
+ return data
+
+ def _parse(img):
+ img = np.asarray(img)
+ if np.issubdtype(img.dtype, np.floating):
+ img = (255 * np.clip(img, 0.0, 1.0)).astype(np.uint8)
+ if img.ndim == 3 and img.shape[0] <= 4 < img.shape[-1]:
+ img = np.transpose(img, (1, 2, 0))
+ return img
+
+ return {**data, "image": {key: _parse(value) for key, value in images.items()}}
+
+
+@dataclasses.dataclass(frozen=True)
+class BuildImageMasks(DataTransformFn):
+ """DataPipeline 改动 (design_v2 D15): 为契约 N 槽相机生成 image_mask (每槽 True)。
+
+ 槽位完备性由契约 key_map 校验保证 (每个声明相机恰好被映射一次), 无需官方
+ aloha_policy 式缺路填黑 + False 分支。缺此 transform 时 Observation.from_dict
+ 取 data["image_mask"] 抛 KeyError。
+ """
+
+ def __call__(self, data: DataDict) -> DataDict:
+ return {**data, "image_mask": {key: np.True_ for key in data["image"]}}
+
+
@dataclasses.dataclass(frozen=True)
class PadStatesAndActions(DataTransformFn):
"""Zero-pads states and actions to the model action dimension."""
--
2.34.1

10
pi_pipeline/__init__.py Normal file
View File

@ -0,0 +1,10 @@
"""OpenPi_Dev 业务包: 契约驱动的 Pi0.5 训练/推理/转换管线 (design_v2)。
模块 (按里程碑交付):
- contract 三层接口契约 (D6/D15): 能力声明 + 派生规格 + 键映射, 校验与序列化
- v3_data LeRobot v3 读取接入 (D1)
- transforms_ext 归一化模式扩展 (D4/D19) 与比例 Resize
- train_runner 超参 + 契约 -> TrainConfig 组装, 训练入口
- serve_xpolicylab XPolicyLab ws 协议 serve 前端 (D7/D16)
- convert_runner j2t 转换入口 (D8/R-10)
"""

389
pi_pipeline/contract.py Normal file
View File

@ -0,0 +1,389 @@
"""三层接口契约 (design_v2 §4.5, D6/D15/D16-D19)。
一份 JSON 契约, 训练配置持有, 并序列化为 checkpoint 内 model_structure.json:
第一层: 用户能力声明 (相机槽 cam1..camN / state 注入 / 语言 / action)
第二层: 派生规格 (代码从第一层算出, 不手填): 每路 token 数 / prefix token 数
第三层: 键映射 key_map (外部数据集键 -> 内部槽位; 图像映射值为列表, 支持一对多)
内部槽名与官方语义名 (base_0_rgb 等) 脱钩 (D15): 槽名是数据字典键而非权重键
(SigLIP 权重全共享), 对官方权重 resume 零影响; 槽数 N 理论无上限 (实际上限
来自显存/序列长度, 契约校验对超大 N 打 WARN 不阻断); cam 序号即 prefix token
拼接顺序, 契约随 checkpoint 锁定, 推理必须同序。
图像输入固定 1:1 (224x224, 16x16 网格, 256 token/路): 非 1:1 比例 (4:3/16:9
及 pos_emb 行切片) 已按决议移除, 模型代码还原为官方状态; 契约中出现的
aspect_ratio 字段仅接受 "1:1" (旧契约兼容), 其余取值非 0 退出。
"""
from __future__ import annotations
import dataclasses
import json
import logging
import re
import sys
from typing import Any
logger = logging.getLogger("openpi.contract")
# 1:1 固定规格: 224x224, patch 14, 16x16 网格 (与官方 model.IMAGE_RESOLUTION 一致)
TOKENS_PER_CAM = 256
NORM_MODES = ("min_max", "mean_std", "vision_only", "quantile")
_SLOT_RE = re.compile(r"^cam(\d+)$")
# 超大 N 只 WARN 不阻断 (D15: 实际上限来自显存/序列长度)
_WARN_CAMERA_COUNT = 8
_ROBOTWIN_TASK_KEY = "task"
def _fail(msg: str) -> None:
"""契约违例: 解析时强制, 非 0 退出且报错可读 (design_v2 §4.5)。"""
sys.exit(f"[contract] {msg}")
@dataclasses.dataclass(frozen=True)
class Contract:
"""解析后的契约。第二层派生量经 property 计算, 不落盘不手填。"""
# 第一层: 用户能力声明
cameras: tuple[str, ...]
state_dim: int
state_inject: bool
language_enabled: bool
max_token_len: int
action_dim: int
action_horizon: int
norm_mode: str
# 第三层: 键映射 (外部键 -> 内部槽位)
key_map: dict[str, Any]
# ------------------------------------------------------------------
# 解析与校验
# ------------------------------------------------------------------
@classmethod
def from_dict(cls, data: dict) -> "Contract":
if not isinstance(data, dict):
_fail(f"contract must be a JSON object, got {type(data).__name__}")
cameras = data.get("cameras")
if not isinstance(cameras, list) or not cameras:
_fail("'cameras' must be a non-empty list")
for cam in cameras:
if not isinstance(cam, str) or not _SLOT_RE.match(cam):
_fail(f"camera slot name must match cam<N>, got {cam!r}")
if len(set(cameras)) != len(cameras):
_fail(f"duplicate camera slots: {cameras}")
cameras = tuple(cameras)
if len(cameras) >= _WARN_CAMERA_COUNT:
logger.warning(
"contract declares %d cameras; prefix ~= %d image tokens + language, "
"memory/attention cost grows superlinearly (accepted, WARN only)",
len(cameras),
len(cameras) * 256,
)
aspect = data.get("aspect_ratio", "1:1")
if aspect != "1:1":
_fail(
f"aspect_ratio {aspect!r} is no longer supported: "
"image input is fixed to 1:1 (224x224); the non-1:1 aspect feature "
"and its model modifications have been removed"
)
state = data.get("state", {})
if not isinstance(state, dict):
_fail("'state' must be an object {dim, inject}")
state_dim = state.get("dim")
if not isinstance(state_dim, int) or state_dim <= 0:
_fail(f"'state.dim' must be a positive int, got {state_dim!r}")
state_inject = state.get("inject", True)
if not isinstance(state_inject, bool):
_fail(f"'state.inject' must be bool, got {state_inject!r}")
language = data.get("language", {})
if not isinstance(language, dict):
_fail("'language' must be an object {enabled, max_token_len}")
language_enabled = language.get("enabled", True)
if not isinstance(language_enabled, bool):
_fail(f"'language.enabled' must be bool, got {language_enabled!r}")
max_token_len = language.get("max_token_len", 200)
if not isinstance(max_token_len, int) or max_token_len <= 0:
_fail(f"'language.max_token_len' must be a positive int, got {max_token_len!r}")
action = data.get("action", {})
if not isinstance(action, dict):
_fail("'action' must be an object {dim, horizon}")
action_dim = action.get("dim")
if not isinstance(action_dim, int) or action_dim <= 0:
_fail(f"'action.dim' must be a positive int, got {action_dim!r}")
action_horizon = action.get("horizon")
if not isinstance(action_horizon, int) or action_horizon <= 0:
_fail(f"'action.horizon' must be a positive int, got {action_horizon!r}")
norm_mode = data.get("norm_mode", "min_max")
if norm_mode not in NORM_MODES:
_fail(f"norm_mode must be one of {NORM_MODES}, got {norm_mode!r}")
if norm_mode == "mean_std" and state_inject:
logger.warning(
"mean_std + state inject: z-score unbounded, clipped to q01/q99 "
"before discretization (D19, aligned with author quantile intent)"
)
if norm_mode == "vision_only" and state_inject:
logger.info("vision_only: state not injected into prompt; prefix deviates "
"from pi05_base pretraining distribution, needs enough finetune steps")
key_map = data.get("key_map")
if not isinstance(key_map, dict) or not key_map:
_fail("'key_map' must be a non-empty object")
contract = cls(
cameras=cameras,
state_dim=state_dim,
state_inject=state_inject,
language_enabled=language_enabled,
max_token_len=max_token_len,
action_dim=action_dim,
action_horizon=action_horizon,
norm_mode=norm_mode,
key_map=dict(key_map),
)
contract._validate_key_map()
return contract
@classmethod
def from_json(cls, text: str) -> "Contract":
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
_fail(f"CONTRACT_JSON is not valid JSON: {exc}")
return cls.from_dict(data)
@classmethod
def from_env(cls, env_var: str = "CONTRACT_JSON") -> "Contract":
import os
raw = os.environ.get(env_var, "").strip()
if not raw:
return cls.robotwin_default()
return cls.from_json(raw)
@classmethod
def from_model_structure(cls, path: str) -> "Contract":
"""从 checkpoint 内 model_structure.json 恢复契约 (serve/convert/量化消费)。"""
try:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
except OSError as exc:
_fail(f"cannot read model_structure.json: {exc}")
if "contract" not in data:
_fail(f"model_structure.json at {path} has no 'contract' field")
return cls.from_dict(data["contract"])
# ------------------------------------------------------------------
# key_map 校验 (D15 规则, 解析时强制)
# ------------------------------------------------------------------
def _validate_key_map(self) -> None:
single_targets: dict[str, str] = {}
slot_sources: dict[str, list[str]] = {}
for source, target in self.key_map.items():
if not isinstance(source, str) or not source:
_fail(f"key_map source must be a non-empty string, got {source!r}")
if source in _RESERVED_TARGET_KEYS:
continue # state/action/prompt 单目标, 在下面统一断言
if isinstance(target, list):
# 图像映射: 值必须是列表 (一对多允许, 同图复用)
for slot in target:
if not isinstance(slot, str):
_fail(f"key_map['{source}'] target list must contain strings, got {slot!r}")
if slot not in self.cameras:
_fail(
f"key_map['{source}'] targets unknown slot {slot!r}; "
f"declared cameras: {list(self.cameras)}"
)
slot_sources.setdefault(slot, []).append(source)
continue
if isinstance(target, str):
if target in self.cameras:
_fail(
f"key_map['{source}'] -> {target!r}: image mapping value must be a LIST "
f"(one-to-many supported), got bare string"
)
single_targets[source] = target
continue
_fail(f"key_map['{source}'] target must be a string or list, got {target!r}")
# 每个声明的 cam 必须恰好被映射一次; 多源 -> 同槽为冲突
for slot in self.cameras:
sources = slot_sources.get(slot, [])
if len(sources) == 0:
_fail(f"camera slot {slot!r} declared but not mapped in key_map")
if len(sources) > 1:
_fail(f"camera slot {slot!r} mapped by multiple sources {sources} (conflict)")
# state/action/prompt 单目标映射
for need, keys in (
("state", ("state/concat", "state", "observation.state")),
("action", ("action/concat", "action", "actions")),
("prompt", ("prompt", "task", "instructions")),
):
hits = [k for k in self.key_map if k in keys]
if len(hits) == 0:
_fail(f"key_map is missing a {need} mapping (expected one of {keys})")
if len(hits) > 1:
_fail(f"key_map has multiple {need} mappings: {hits} (single target only)")
# ------------------------------------------------------------------
# 第二层: 派生规格 (property 计算, 不手填)
# ------------------------------------------------------------------
@property
def per_cam_resolution(self) -> tuple[int, int]:
"""固定 1:1: (224, 224)。保留 property 以兼容消费方调用点。"""
return (224, 224)
@property
def tokens_per_cam(self) -> int:
"""固定 1:1: 16x16 网格 256 token。"""
return TOKENS_PER_CAM
@property
def num_cameras(self) -> int:
return len(self.cameras)
@property
def prefix_tokens(self) -> int:
return self.num_cameras * self.tokens_per_cam + self.max_token_len
# ------------------------------------------------------------------
# 键映射视图 (消费方: repack / serve 逆映射 / 量化校验)
# ------------------------------------------------------------------
@property
def vision_map(self) -> dict[str, tuple[str, ...]]:
"""外部图像键 -> 槽位元组 (一对多)。"""
return {
src: tuple(target)
for src, target in self.key_map.items()
if isinstance(target, list)
}
@property
def state_source(self) -> str:
hits = [k for k in ("state/concat", "state", "observation.state") if k in self.key_map]
return self.key_map[hits[0]] if hits else "state"
@property
def action_source(self) -> str:
hits = [k for k in ("action/concat", "action", "actions") if k in self.key_map]
return self.key_map[hits[0]] if hits else "actions"
@property
def prompt_source(self) -> str:
hits = [k for k in ("prompt", "task", "instructions") if k in self.key_map]
return self.key_map[hits[0]] if hits else "prompt"
def inverse_vision_map(self) -> dict[str, str]:
"""槽位 -> 外部键 (serve 侧: 评测 obs 对外键 -> 内部槽)。"""
inverse = {}
for source, slots in self.vision_map.items():
for slot in slots:
inverse[slot] = source
return inverse
# ------------------------------------------------------------------
# 模型配置参数 (Pi0Config 契约化字段)
# ------------------------------------------------------------------
@property
def discrete_state_input(self) -> bool:
if self.norm_mode == "vision_only":
return False
return self.state_inject
@property
def use_quantile_norm(self) -> bool:
return self.norm_mode == "quantile"
def pi0_config_kwargs(self) -> dict[str, Any]:
"""Pi0Config 契约化字段 (image_keys + 归一化联动; 分辨率固定 224x224 官方值)。"""
return {
"image_keys": tuple(self.cameras),
"action_dim": self.action_dim,
"action_horizon": self.action_horizon,
"max_token_len": self.max_token_len,
"pi05": True,
"discrete_state_input": self.discrete_state_input,
}
# ------------------------------------------------------------------
# 序列化 (checkpoint 内 model_structure.json 锚定, D13/R-9)
# ------------------------------------------------------------------
def to_model_structure(self) -> dict:
"""model_structure.json 内容: 契约全文 + 派生规格快照 (浮点结构变更的量化锚点)。"""
return {
"contract": self.to_dict(),
"derived": {
"per_cam_resolution": list(self.per_cam_resolution),
"tokens_per_cam": self.tokens_per_cam,
"prefix_tokens": self.prefix_tokens,
"num_cameras": self.num_cameras,
"discrete_state_input": self.discrete_state_input,
"use_quantile_norm": self.use_quantile_norm,
},
}
def to_dict(self) -> dict:
return {
"cameras": list(self.cameras),
"state": {"dim": self.state_dim, "inject": self.state_inject},
"language": {"enabled": self.language_enabled, "max_token_len": self.max_token_len},
"action": {"dim": self.action_dim, "horizon": self.action_horizon},
"norm_mode": self.norm_mode,
"key_map": dict(self.key_map),
}
def to_json(self) -> str:
return json.dumps(self.to_model_structure(), ensure_ascii=False, indent=2)
# ------------------------------------------------------------------
# 出厂默认 (RoboTwin 四相机 -> cam1..cam4)
# ------------------------------------------------------------------
@classmethod
def robotwin_default(cls) -> "Contract":
"""RoboTwin 数据集默认契约: 四相机 1:1, 14 维 state/action, min_max 归一化。
外部键 = data2lerobot 转换产物键 (observation.images.* / observation.state /
action / task)。
"""
return cls.from_dict(
{
"cameras": ["cam1", "cam2", "cam3", "cam4"],
"state": {"dim": 14, "inject": True},
"language": {"enabled": True, "max_token_len": 200},
"action": {"dim": 14, "horizon": 10},
"norm_mode": "min_max",
"key_map": {
"observation.images.cam_high": ["cam1"],
"observation.images.cam_third_view": ["cam2"],
"observation.images.cam_left_wrist": ["cam3"],
"observation.images.cam_right_wrist": ["cam4"],
"observation.state": "state",
"action": "actions",
_ROBOTWIN_TASK_KEY: "prompt",
},
}
)
_RESERVED_TARGET_KEYS = ("observation.state", "action", "actions", "prompt", "task", "instructions")

View File

@ -0,0 +1,209 @@
"""j2t 转换入口 (design_v2 §8, D8/R-10)。
JAX checkpoint (契约管线产物, 含 model_structure.json) -> torch model.safetensors。
改造 (相对 examples/convert_jax_model_to_pytorch.py):
- 结构判定读 model_structure.json (契约快照), 不再用 config 名 / 目录名字符串
(R-10: examples 内三处 "pi05" in checkpoint_dir 已改为 pi05 形参, 本入口以
契约字段驱动);
- 转换前结构校验 (params/ 存在 + 契约可恢复);
- 转换后抽样数值一致性自检 (JAX vs torch 前向对拍: 同一 fake_obs 与噪声,
actions 余弦相似度 / 最大绝对误差), 结果写进产物 metadata;
- 产物附加: model_structure.json 透传 + paligemma_tokenizer.model 拷贝
(下游消费: model.safetensors + tokenizer 同目录)。
超参: CHECKPOINT_PATH (与上游端口 checkpoint 二选一上游优先), OUTPUT 精度
(bfloat16 默认)。
"""
from __future__ import annotations
import importlib.util
import json
import logging
import os
import shutil
import sys
import types
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as _contract # noqa: E402
from ws_common import report_utils # noqa: E402
def locate_checkpoint() -> Path:
"""上游端口 (portId=checkpoint) 优先, CHECKPOINT_PATH 兜底。
与 serve_xpolicylab.locate_checkpoint 同语义; 此处独立实现避免顶层引入
XPolicyLab client_server 依赖 (convert 场景无需 serve 栈)。
"""
import json
artifacts_path = os.environ.get("INPUT_ARTIFACTS_PATH", "")
if artifacts_path and os.path.isfile(artifacts_path):
with open(artifacts_path, "r", encoding="utf-8") as f:
entries = json.load(f) if artifacts_path else []
for entry in entries or []:
if entry.get("portId") == "checkpoint" and not str(entry.get("path", "")).startswith("bos://"):
return Path(entry["path"])
value = os.environ.get("CHECKPOINT_PATH", "").strip()
if value:
return Path(value)
sys.exit("[input] neither upstream checkpoint port nor CHECKPOINT_PATH is set")
logger = logging.getLogger("openpi.convert")
_CONVERT_SCRIPT = Path(__file__).resolve().parents[1] / "openpi" / "examples" / "convert_jax_model_to_pytorch.py"
def _load_convert_module():
spec = importlib.util.spec_from_file_location("openpi_convert", _CONVERT_SCRIPT)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def numeric_consistency_check(checkpoint_dir: Path, contract: _contract.Contract, torch_output: Path) -> dict:
"""JAX vs torch 前向对拍: 同一 obs + 噪声, 比较 sample_actions。"""
import jax
import torch
import openpi.models.model as _model
import dataclasses
from openpi.models.pi0_config import Pi0Config
# 自检单次前向, 关闭 torch.compile (max-autotune 首编 10min+ 且对拍数值无益)
model_cfg = dataclasses.replace(
Pi0Config(**contract.pi0_config_kwargs()), pytorch_compile_mode=None
)
jax_model = model_cfg.load(_model.restore_params(checkpoint_dir / "params", dtype="bfloat16"))
# load_pytorch 按 train_config.model 取模型配置 (官方 scripts 惯例), 此处以
# shim 适配独立调用
train_cfg_shim = types.SimpleNamespace(model=model_cfg)
torch_model = model_cfg.load_pytorch(train_cfg_shim, str(torch_output / "model.safetensors"))
torch_model = torch_model.to("cuda" if torch.cuda.is_available() else "cpu").eval()
device = next(torch_model.parameters()).device
obs_spec, _ = model_cfg.inputs_spec()
obs = jax.tree.map(
lambda s: np.random.uniform(-1.0, 1.0, size=s.shape).astype(np.float32)
if s.dtype == np.float32 or str(s.dtype) == "float32"
else (np.random.uniform(0, 2, size=s.shape).astype(np.int32) if "int" in str(s.dtype) else np.zeros(s.shape, bool)),
obs_spec,
)
rng = np.random.default_rng(0)
noise = rng.normal(size=(1, contract.action_horizon, model_cfg.action_dim)).astype(np.float32)
# jaxtyping>=0.3 的 Array hint 不含 numpy 数组, jax 侧统一转 jnp
import jax.numpy as jnp
jax_obs = jax.tree.map(jnp.asarray, obs)
jax_actions = np.asarray(jax_model.sample_actions(jax.random.key(0), jax_obs, noise=jnp.asarray(noise)))
torch_obs = jax.tree.map(lambda x: torch.from_numpy(np.asarray(x)).to(device), obs)
# torch 栈图像约定 CHW (官方 train_pytorch 数据管线/transformers get_image_features),
# jax 侧 inputs_spec 为 HWC — 图像叶 permute
for cam in list(torch_obs.images.keys()):
torch_obs.images[cam] = torch.permute(torch_obs.images[cam], (0, 3, 1, 2))
torch_noise = torch.from_numpy(noise).to(device)
with torch.no_grad():
torch_actions = np.asarray(torch_model.sample_actions(device, torch_obs, noise=torch_noise).cpu())
a, b = jax_actions.reshape(-1), torch_actions.reshape(-1)
cos = float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9))
max_abs = float(np.abs(a - b).max())
return {"cosine": cos, "max_abs_diff": max_abs, "shape": list(jax_actions.shape)}
def main() -> None:
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
if not output_dir:
sys.exit("[hyperparam] OUTPUT_DIR not set")
checkpoint_dir = locate_checkpoint()
contract = load_contract_checked(checkpoint_dir)
precision = os.environ.get("PRECISION", "bfloat16").strip() or "bfloat16"
logger.info("[convert] JAX %s -> torch (%s)", checkpoint_dir, precision)
convert = _load_convert_module()
from openpi.models.pi0_config import Pi0Config
model_cfg = Pi0Config(**contract.pi0_config_kwargs())
torch_output = output_dir / "checkpoint_torch"
convert.convert_pi0_checkpoint(str(checkpoint_dir), precision, str(torch_output), model_cfg)
# 契约快照透传 (checkpoint 自带或 CONTRACT_JSON 注入时由当前契约生成)
source_structure = checkpoint_dir / "model_structure.json"
if source_structure.is_file():
shutil.copy2(source_structure, torch_output / "model_structure.json")
else:
(torch_output / "model_structure.json").write_text(contract.to_json(), encoding="utf-8")
logger.info("[convert] contract snapshot generated from CONTRACT_JSON injection")
# tokenizer 随行 (量化交接)
tokenizer_src = _find_tokenizer()
if tokenizer_src:
shutil.copy2(tokenizer_src, torch_output / "paligemma_tokenizer.model")
logger.info("[convert] tokenizer copied: %s", tokenizer_src)
else:
logger.warning(
"[convert] paligemma_tokenizer.model not found in cache; quantization handoff "
"requires it next to model.safetensors (run serve/train once or place manually)"
)
# 数值一致性自检
check = numeric_consistency_check(checkpoint_dir, contract, torch_output)
verdict = check["cosine"] >= 0.95
check["verdict"] = "pass" if verdict else "fail"
logger.info("[convert] numeric check: cos=%.4f max_abs=%.4f -> %s", check["cosine"], check["max_abs_diff"], check["verdict"])
(torch_output / "convert_metadata.json").write_text(
json.dumps({"numeric_check": check, "precision": precision, "source_checkpoint": str(checkpoint_dir)},
ensure_ascii=False, indent=2),
encoding="utf-8",
)
report_utils.write_artifacts(
output_dir,
[{"name": "checkpoint_torch", "type": "file", "portId": "checkpoint", "path": str(torch_output)}],
)
if not verdict:
sys.exit("[convert] numeric consistency check FAILED (cosine < 0.95)")
def load_contract_checked(checkpoint_dir: Path) -> _contract.Contract:
"""契约来源优先级: checkpoint 内 model_structure.json > CONTRACT_JSON 超参注入
(官方权重/外部权重无契约快照时, 如 RoboTwin cotrain 权重, T5)。"""
structure = checkpoint_dir / "model_structure.json"
if structure.is_file():
return _contract.Contract.from_model_structure(str(structure))
raw = os.environ.get("CONTRACT_JSON", "").strip()
if raw:
logger.info("[convert] model_structure.json not found; using CONTRACT_JSON injection")
# 快照只落产物侧 (main 透传), 不回写源 checkpoint 目录 (外部资产可能只读)
return _contract.Contract.from_json(raw)
sys.exit(
f"[convert] {structure} not found and CONTRACT_JSON not set; provide either a "
"contract-pipeline checkpoint or an explicit CONTRACT_JSON"
)
def _find_tokenizer() -> Path | None:
"""openpi 缓存中的 paligemma tokenizer (量化交接)。"""
candidates = []
data_home = os.environ.get("OPENPI_DATA_HOME", "")
if data_home:
candidates.append(Path(data_home) / "big_vision" / "paligemma_tokenizer.model")
candidates.append(Path.home() / ".cache" / "openpi" / "big_vision" / "paligemma_tokenizer.model")
for cand in candidates:
if cand.is_file():
return cand
return None
if __name__ == "__main__":
main()

View File

@ -0,0 +1,111 @@
"""norm_stats 独立统计入口 (design_v2 §6, D11)。
统计语义与训练自动补算一致: repack + data_transforms 空间 (不含 Normalize /
model_transforms), 键固定 state/actions; RunningStats 含真 min/max (D4)。
产物: $OUTPUT_DIR/norm_stats/norm_stats.json。
超参: DATASET_PATH (与上游端口二选一上游优先), CONTRACT_JSON (决定 repack 与
统计维度), MAX_FRAMES (可选采样上限, 0 = 全量)。
"""
from __future__ import annotations
import logging
import os
import sys
from pathlib import Path
import numpy as np
import openpi.shared.normalize as _normalize
import openpi.training.data_loader as _data_loader
import openpi.training.config as _config
import openpi.transforms as _transforms
from openpi.models.pi0_config import Pi0Config
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as _contract # noqa: E402
from pi_pipeline.train_runner import ContractDataConfigFactory, resolve_dataset_paths # noqa: E402
from ws_common import report_utils # noqa: E402
def _env_int(key: str, default: int) -> int:
raw = os.environ.get(key, "").strip()
if not raw:
return default
try:
return int(raw)
except ValueError:
sys.exit(f"[hyperparam] {key}: invalid int value {raw!r}")
def main() -> None:
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
if not output_dir:
sys.exit("[hyperparam] OUTPUT_DIR not set")
contract = _contract.Contract.from_env()
dataset_paths = resolve_dataset_paths()
# LeRobot v3 本地读取: HF_LEROBOT_HOME 指向数据集公共父目录 (多数据集共父,
# 已由 resolve_dataset_paths 校验), repo_id = 目录名。
# 入口 norm_stats.sh 预设的 $OUTPUT_DIR 值必须被覆盖 (强制赋值, 非 setdefault);
# lerobot 常量 import 时固化且 lerobot_dataset.py 以 from-import 持有副本,
# 环境变量对已加载模块无效, 需双处 patch (语义同 train_runner.run)。
lerobot_home = dataset_paths[0].parent
os.environ["HF_LEROBOT_HOME"] = str(lerobot_home)
import lerobot.datasets.lerobot_dataset as _ld
import lerobot.utils.constants as _lc
_lc.HF_LEROBOT_HOME = lerobot_home
_ld.HF_LEROBOT_HOME = lerobot_home
config = _config.TrainConfig(
name="norm_stats",
model=Pi0Config(**contract.pi0_config_kwargs()),
data=ContractDataConfigFactory(dataset_paths, contract, output_dir / "assets"),
)
# 多数据集: 全量混合上计算共享统计 (与训练自动补算同语义)
stats = {"state": _normalize.RunningStats(), "actions": _normalize.RunningStats()}
max_frames = _env_int("MAX_FRAMES", 0)
done_frames = 0
for data_config in config.data.dataset_configs(config.model):
dataset = _data_loader.create_torch_dataset(data_config, contract.action_horizon, config.model)
dataset = _data_loader.TransformedDataset(
dataset,
[
*data_config.repack_transforms.inputs,
*data_config.data_transforms.inputs,
_transforms.InjectDefaultPrompt(None),
],
)
total = len(dataset)
indices = range(total)
if max_frames and max_frames < total:
rng = np.random.default_rng(0)
indices = sorted(rng.choice(total, size=max_frames, replace=False))
for i, idx in enumerate(indices):
item = dataset[idx]
stats["state"].update(np.asarray(item["state"])[None, :])
stats["actions"].update(np.asarray(item["actions"]))
if (done_frames + i + 1) % 500 == 0:
print(f"DROBOTICFLOW_PROGRESS {{\"stage\":\"norm_stats\",\"frames\":{done_frames + i + 1},\"dataset\":\"{data_config.repo_id}\"}}", flush=True)
logging.info("norm_stats %s %d/%d", data_config.repo_id, i + 1, len(indices))
done_frames += len(indices)
del dataset
out_dir = output_dir / "norm_stats"
_normalize.save(out_dir, {k: v.get_statistics() for k, v in stats.items()})
logging.info("[norm_stats] saved (%d sampled frames): %s", done_frames, out_dir)
report_utils.write_artifacts(
output_dir,
[{"name": "norm_stats", "type": "file", "portId": "norm_stats", "path": str(out_dir)}],
)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,434 @@
"""XPolicyLab ws 协议 serve 前端 (design_v2 §7, D7/D16)。
实现 XPolicyLab ModelTemplate 接口 (update_obs / get_action / reset /
prepare_case / on_trial_end), 包装 openpi Policy; 挂 XPolicyLab PolicyServer
(ws/msgpack) 对外服务。无并发需求 (D16): 单客户端串行, action chunk 整块返回
(官方 infer 语义)。
obs 契约 (RoboTwin 评测客户端 robotwin_obs_to_xpolicylab):
obs["vision"][cam]["color"] uint8 RGB (H,W,3) / JPEG buffer, cam 为对外键
(cam_head / cam_third_view / cam_left_wrist /
cam_right_wrist)
obs["state"]["{left,right}_{arm,ee}_joint_state"] 关节状态
obs["instruction"] 语言指令
协议键 -> 内部槽位: 经 PROTOCOL_TO_DATASET 别名 + 契约 key_map 逆映射
(评测 obs 的相机集合与采集同源, 缺路补零 + mask False, 未声明相机忽略 + INFO)。
state 拼接顺序 [左臂6, 左爪1, 右臂6, 右爪1] 与 data2lerobot concat 伪键一致;
action 输出同序截断到契约 state_dim (joint 模式)。
入口: python serve_xpolicylab.py (entry/serve_jax.sh | serve_torch.sh 调用)
超参: CHECKPOINT_PATH (与上游端口 checkpoint 二选一上游优先), SERVE_HOST,
SERVE_PORT, ACTION_TYPE (joint|ee)。
"""
from __future__ import annotations
import asyncio
import dataclasses
import logging
import os
import pathlib
import sys
import time
from pathlib import Path
import numpy as np
import openpi.models.model as _model
import openpi.policies.policy as _policy
import openpi.shared.normalize as _normalize
import openpi.training.checkpoints as _checkpoints
import openpi.transforms as _transforms
from openpi.models.pi0_config import Pi0Config
from openpi.models.tokenizer import PaligemmaTokenizer
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as _contract # noqa: E402
from pi_pipeline.train_runner import ContractDataConfigFactory # noqa: E402
from ws_common import report_utils # noqa: E402
# XPolicyLab 服务端 (vendored 到镜像 /app/src/src-externals/XPolicyLab)
_XPOLICYLAB_ROOT = Path(os.environ.get("XPOLICYLAB_ROOT", "/app/src/src-externals/XPolicyLab"))
for _p in (str(_XPOLICYLAB_ROOT), str(_XPOLICYLAB_ROOT / "integrations")):
if _p not in sys.path:
sys.path.insert(0, _p)
from client_server.ws.model_server import PolicyServer, PolicyServerConfig # noqa: E402
logger = logging.getLogger("openpi.serve")
# ADAPT_TO_PI (环境变量开关): 权重训练侧使用 openpi LeRobotAlohaDataConfig 且
# adapt_to_pi=True 时必须开启 (RoboTwin 官方 cotrain ckpt 即此配置)。语义拷贝自
# openpi/policies/aloha_policy.py: joint_flip_mask + gripper 线性位置 -> pi 角度
# 空间变换; 输入侧 raw->pi (AlohaInputs), 输出侧 pi->raw (AlohaOutputs)。
ADAPT_TO_PI = os.environ.get("ADAPT_TO_PI", "").strip().lower() in ("1", "true", "yes")
def _joint_flip_mask() -> np.ndarray:
return np.array([1, -1, -1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1], dtype=np.float32)
def _gripper_to_angular(value: np.ndarray) -> np.ndarray:
value = value * (0.05800 - 0.01844) + 0.01844 # unnormalize [0,1] -> 线性位置
v = (0.022**2 + value**2 - 0.036**2) / (2 * 0.022 * value)
value = np.arcsin(np.clip(v, -1.0, 1.0)) # linear_to_radian(arm=0.036, horn=0.022)
return (value - 0.5476) / (1.6296 - 0.5476) # normalize 到 pi 爪空间
def _gripper_from_angular(value: np.ndarray) -> np.ndarray:
value = value + 0.5476
return (value - (-0.6213)) / (1.4910 - (-0.6213))
def _adapt_state_pi(state: np.ndarray) -> np.ndarray:
"""AlohaInputs._decode_state(adapt_to_pi=True) 同语义。"""
out = _joint_flip_mask() * state
out[[6, 13]] = _gripper_to_angular(out[[6, 13]])
return out
def _unadapt_actions_pi(actions: np.ndarray) -> np.ndarray:
"""AlohaOutputs._encode_actions(adapt_to_pi=True) 同语义 (chunk 版)。"""
out = _joint_flip_mask() * actions
out[..., [6, 13]] = _gripper_from_angular(out[..., [6, 13]])
return out
# RoboTwin 评测协议对外键 -> data2lerobot 数据集键 (XPolicyLab 转换器 KEY_MAP 约定)
PROTOCOL_TO_DATASET = {
"cam_head": "observation.images.cam_high",
"cam_third_view": "observation.images.cam_third_view",
"cam_left_wrist": "observation.images.cam_left_wrist",
"cam_right_wrist": "observation.images.cam_right_wrist",
}
# state 拼接顺序 (与 data2lerobot concat 伪键一致: 左臂, 左爪, 右臂, 右爪)
STATE_KEYS = (
"left_arm_joint_state",
"left_ee_joint_state",
"right_arm_joint_state",
"right_ee_joint_state",
)
def locate_checkpoint() -> Path:
"""上游端口 (portId=checkpoint) 优先, CHECKPOINT_PATH 兜底。"""
import json
artifacts_path = os.environ.get("INPUT_ARTIFACTS_PATH", "")
if artifacts_path and os.path.isfile(artifacts_path):
with open(artifacts_path, "r", encoding="utf-8") as f:
entries = json.load(f) if artifacts_path else []
for entry in entries or []:
if entry.get("portId") == "checkpoint" and not str(entry.get("path", "")).startswith("bos://"):
return Path(entry["path"])
value = os.environ.get("CHECKPOINT_PATH", "").strip()
if value:
return Path(value)
sys.exit("[input] neither upstream checkpoint port nor CHECKPOINT_PATH is set")
def load_contract(checkpoint_dir: Path) -> _contract.Contract:
structure = checkpoint_dir / "model_structure.json"
if structure.is_file():
return _contract.Contract.from_model_structure(str(structure))
# 官方权重/外部权重无契约快照时 CONTRACT_JSON 注入 (与 convert 同语义)
raw = os.environ.get("CONTRACT_JSON", "").strip()
if raw:
logger.info("[contract] using CONTRACT_JSON injection")
return _contract.Contract.from_json(raw)
sys.exit(
f"[contract] {structure} not found: checkpoint was not produced by the "
"contract pipeline (retrain, attach model_structure.json, or set CONTRACT_JSON)"
)
def build_policy(checkpoint_dir: Path, contract: _contract.Contract) -> _policy.Policy:
"""契约驱动组装 openpi Policy (对齐训练侧 transform 序列)。"""
weight_path = checkpoint_dir / "model.safetensors"
is_pytorch = weight_path.is_file()
# torch serve: 单流推理关 torch.compile (max-autotune 首编 10min+; 可经
# TORCH_COMPILE=true 恢复)
if os.environ.get("TORCH_COMPILE", "").strip().lower() not in ("1", "true", "yes"):
model_cfg = dataclasses.replace(Pi0Config(**contract.pi0_config_kwargs()), pytorch_compile_mode=None)
else:
model_cfg = Pi0Config(**contract.pi0_config_kwargs())
if is_pytorch:
import openpi.training.config as _config # noqa: F401
# load_pytorch 按 train_config.model 取模型配置 (官方 scripts 惯例), shim 适配
import types as _types
model = model_cfg.load_pytorch(_types.SimpleNamespace(model=model_cfg), str(weight_path))
# TORCH_SERVE_DTYPE=fp32 跳过 bf16 选择性转换 (默认 bf16; 长程去噪推理对
# bf16 舍入敏感, fp32 与 jax 数值一致性最好但延迟更高)
if os.environ.get("TORCH_SERVE_DTYPE", "").strip().lower() != "fp32":
model.paligemma_with_expert.to_bfloat16_for_selected_params("bfloat16")
else:
model = model_cfg.load(_model.restore_params(checkpoint_dir / "params", dtype="bfloat16"))
# norm_stats: checkpoint assets 优先 (与训练一致), 契约 asset_id = 数据集目录名不可知,
# 扫描 assets/ 下唯一子目录
assets_root = checkpoint_dir / "assets"
if not assets_root.is_dir():
sys.exit(f"[norm_stats] checkpoint assets not found: {assets_root}")
asset_dirs = [d for d in sorted(assets_root.iterdir()) if d.is_dir()]
if len(asset_dirs) != 1:
sys.exit(f"[norm_stats] expected exactly one asset dir under {assets_root}, got {asset_dirs}")
norm_stats = _checkpoints.load_norm_stats(assets_root, asset_dirs[0].name)
h, w = contract.per_cam_resolution
model_transforms = _transforms.Group(
inputs=[
_transforms.ResizeImages(h, w),
_transforms.TokenizePrompt(
PaligemmaTokenizer(contract.max_token_len),
discrete_state_input=contract.discrete_state_input,
),
_transforms.PadStatesAndActions(model_cfg.action_dim),
]
)
pytorch_device = None
if is_pytorch:
import torch
pytorch_device = "cuda" if torch.cuda.is_available() else "cpu"
return _policy.Policy(
model,
# 输入 repack 在 OpenPiModelTemplate._convert_obs 中手工完成 (评测 obs 的
# vision/state 嵌套结构与训练 repack 输入不同), 此处从 Normalize 起。
transforms=[
_transforms.Normalize(
norm_stats,
use_quantiles=contract.use_quantile_norm,
mode=None if contract.norm_mode == "quantile" else contract.norm_mode,
clip_to_quantiles=contract.norm_mode == "mean_std",
),
*model_transforms.inputs,
],
output_transforms=[
_transforms.Unnormalize(
norm_stats,
use_quantiles=contract.use_quantile_norm,
mode=None if contract.norm_mode == "quantile" else contract.norm_mode,
),
],
metadata={"contract": contract.to_dict(), "prefix_tokens": contract.prefix_tokens},
is_pytorch=is_pytorch,
pytorch_device=pytorch_device,
)
class OpenPiModelTemplate:
"""XPolicyLab ModelTemplate 兼容前端 (单客户端串行, D16)。"""
def __init__(self, policy: _policy.Policy, contract: _contract.Contract):
self._policy = policy
self._contract = contract
self._latest_obs: dict | None = None
self._infer_cnt = 0
self._last_state: np.ndarray | None = None
# 协议相机键 -> 内部槽位 (inverse_vision_map 返回 槽位->数据集键, 此处反转)
dataset_to_slot = {src: slot for slot, src in contract.inverse_vision_map().items()}
self._protocol_to_slot: dict[str, str] = {}
for proto_key, dataset_key in PROTOCOL_TO_DATASET.items():
if dataset_key in dataset_to_slot:
self._protocol_to_slot[proto_key] = dataset_to_slot[dataset_key]
if not self._protocol_to_slot:
sys.exit("[contract] no protocol camera maps to contract slots; check key_map")
# -- ModelTemplate 接口 ------------------------------------------------
def update_obs(self, obs: dict) -> None:
self._latest_obs = obs
def update_obs_batch(self, obs_list: list[dict]) -> None:
self._latest_obs = obs_list[-1] if obs_list else None
def get_action(self):
if self._latest_obs is None:
raise RuntimeError("get_action called before update_obs")
# SERVE_DELAY_MS: 受控延迟注入 (仅延迟敏感度实验用, 生产不设)
delay_ms = os.environ.get("SERVE_DELAY_MS", "").strip()
if delay_ms:
time.sleep(float(delay_ms) / 1000.0)
inputs = self._convert_obs(self._latest_obs)
# W3 数值对拍支持: SERVE_NOISE_SEED 设置时噪声确定化 (seed+infer 计数,
# 与伪量化/板端服务用同一 numpy rng 生成同一数组, 服务级同噪对拍)
infer_kwargs = {}
seed_env = os.environ.get("SERVE_NOISE_SEED", "").strip()
if seed_env:
rng = np.random.default_rng(int(seed_env) + self._infer_cnt)
infer_kwargs["noise"] = rng.normal(
size=(self._contract.action_horizon, self._contract.action_dim)
).astype(np.float32)
result = self._policy.infer(inputs, **infer_kwargs)
actions = np.asarray(result["actions"])
# 截断到契约维 (joint 模式: [左臂6, 左爪1, 右臂6, 右爪1]; model 可能 pad 到
# action_dim)。整块返回 (H, dim), 客户端 normalize_action_chunk 摊平逐动作。
dim = min(self._contract.state_dim, actions.shape[-1])
actions = actions[..., :dim]
# DELTA_ACTION_SPACE=true: 模型动作空间为 delta joint (官方 aloha cotrain
# use_delta_joint_actions 语义, 官方 outputs=AbsoluteActions 还原) — 臂 6 维
# 加当前 state, 爪 (第 7/14 维) 绝对值直通。ADAPT_TO_PI 时 state 为 pi 空间
# (还原后统一转回 raw 空间输出)
if os.environ.get("DELTA_ACTION_SPACE", "").strip().lower() in ("1", "true", "yes"):
state_pi = getattr(self, "_last_state_pi", None)
if state_pi is None:
state_pi = self._last_state
if state_pi is not None:
delta_mask = np.ones(dim, dtype=np.float32)
delta_mask[6] = 0.0
delta_mask[13] = 0.0
actions = actions + state_pi[None, :dim] * delta_mask[None, :dim]
if ADAPT_TO_PI:
# AlohaOutputs: pi -> raw (flip + gripper 角度逆变换), 官方输出链末端
actions = _unadapt_actions_pi(actions).astype(np.float32)
if self._infer_cnt < 3 or self._infer_cnt % 100 == 0:
logger.info(
"[infer#%d] action[0]=%s absmax=%.3f", self._infer_cnt, np.round(actions[0, :dim], 3), float(np.abs(actions[..., :dim]).max())
)
self._infer_cnt += 1
return actions
def _capture_state(self, state: np.ndarray) -> None:
self._last_state = state
def get_action_batch(self, env_idx_list):
return self.get_action()
def reset(self) -> None:
self._latest_obs = None
def prepare_case(self, case_meta: dict) -> None:
logger.info("prepare_case: %s", {k: case_meta.get(k) for k in ("task_name", "seed", "action_type")})
def on_trial_end(self, result: dict) -> None:
logger.info("trial_end: %s", result)
# -- obs 转换 ------------------------------------------------------------
def _convert_obs(self, obs: dict) -> dict:
c = self._contract
vision = obs.get("vision", {})
image = {}
image_mask = {}
for slot in c.cameras:
proto_key = next((k for k, v in self._protocol_to_slot.items() if v == slot), None)
cam_data = vision.get(proto_key) if proto_key else None
color = cam_data.get("color") if isinstance(cam_data, dict) else None
if color is not None:
arr = np.asarray(color)
if arr.dtype != np.uint8:
arr = np.clip(arr * 255.0 if arr.max() <= 1.0 else arr, 0, 255).astype(np.uint8)
if arr.ndim == 3 and arr.shape[0] <= 4 and arr.shape[-1] != 3 and arr.shape[0] == 3:
arr = np.transpose(arr, (1, 2, 0)) # CHW -> WHC 容错
image[slot] = arr
image_mask[slot] = np.bool_(True)
else:
logger.info("camera %s (slot %s) absent in obs: zero-filled + masked", proto_key, slot)
h, w = c.per_cam_resolution
image[slot] = np.zeros((h, w, 3), dtype=np.uint8)
image_mask[slot] = np.bool_(False)
for proto_key in vision:
if proto_key not in self._protocol_to_slot:
logger.info("obs camera %s not in contract key_map: ignored", proto_key)
state_parts = []
state_in = obs.get("state", {})
for key in STATE_KEYS:
value = state_in.get(key)
if value is None:
sys.exit(f"[obs] state key {key!r} missing in obs: {sorted(state_in)}")
state_parts.append(np.atleast_1d(np.asarray(value, dtype=np.float32)).reshape(-1))
state = np.concatenate(state_parts)
if state.shape[0] < c.state_dim:
state = np.concatenate([state, np.zeros(c.state_dim - state.shape[0], dtype=np.float32)])
elif state.shape[0] > c.state_dim:
state = state[: c.state_dim]
# ADAPT_TO_PI: state 进归一化/离散化前转 pi 空间 (AlohaInputs 语义);
# delta 还原在 pi 空间做, 最后输出再转回 raw (AlohaOutputs 语义)
self._last_state_pi = _adapt_state_pi(state).astype(np.float32) if ADAPT_TO_PI else state.astype(np.float32)
state = self._last_state_pi
prompt_value = obs.get("instruction")
if not prompt_value:
instructions = obs.get("instructions")
if isinstance(instructions, list) and instructions:
prompt_value = instructions[0]
prompt = str(prompt_value or "")
self._capture_state(state)
if os.environ.get("SERVE_DUMP_DIR", "").strip():
import hashlib
dump_dir = pathlib.Path(os.environ["SERVE_DUMP_DIR"])
dump_dir.mkdir(parents=True, exist_ok=True)
key = hashlib.md5(str(obs.get("instruction", ""))[:64].encode()).hexdigest()[:6]
np.savez_compressed(
dump_dir / f"obs_{self._infer_cnt:04d}_{key}.npz",
**{f"img_{k}": v for k, v in image.items()},
state=state,
prompt=np.array(prompt),
)
return {"image": image, "image_mask": image_mask, "state": state, "prompt": prompt}
async def _run_server(model, host: str, port: int) -> None:
server = PolicyServer(model, PolicyServerConfig(host=host, port=port))
await server.start()
logger.info("serving on %s (ws protocol, single client serial)", server.url)
await server.serve_forever()
def main() -> None:
logging.basicConfig(level=logging.INFO, force=True, format="%(asctime)s [%(levelname)s] %(message)s")
checkpoint_dir = locate_checkpoint()
contract = load_contract(checkpoint_dir)
logger.info(
"[contract] cameras=%s resolution=%s prefix_tokens=%d norm=%s",
contract.cameras, contract.per_cam_resolution,
contract.prefix_tokens, contract.norm_mode,
)
policy = build_policy(checkpoint_dir, contract)
model = OpenPiModelTemplate(policy, contract)
host = os.environ.get("SERVE_HOST", "0.0.0.0")
port = int(os.environ.get("SERVE_PORT", "19000"))
# serve 不注册算子 (D7): 不走收尾协议, 常驻服务; 产物清单仅记录 serve 元信息
output_dir = os.environ.get("OUTPUT_DIR", "")
if output_dir:
report_utils.write_artifacts(
output_dir,
[{"name": "serve_meta", "type": "json", "portId": "serve_meta",
"path": _write_serve_meta(output_dir, contract, host, port)}],
)
asyncio.run(_run_server(model, host, port))
def _write_serve_meta(output_dir: str, contract: _contract.Contract, host: str, port: int) -> str:
import json
path = Path(output_dir) / "serve_meta.json"
path.write_text(
json.dumps(
{"host": host, "port": port, "protocol": "ws", "contract": contract.to_dict()},
ensure_ascii=False, indent=2,
),
encoding="utf-8",
)
return str(path)
if __name__ == "__main__":
main()

613
pi_pipeline/train_runner.py Normal file
View File

@ -0,0 +1,613 @@
# 来源: openpi 仓库 scripts/train.py @ 15a9616 (JAX 训练主循环拷贝改造)
# 改动点:
# 1) 入口从 tyro CLI 配置名改为超参环境变量 + CONTRACT_JSON 契约 (pi_pipeline.contract),
# TrainConfig 由 build_train_config() 程序化组装, 不注册进 _CONFIGS
# 2) wandb 强制关闭 (design_v2 §10.2: 离线镜像 wandb 禁用), 首批图像 wandb 日志移除
# 3) 训练循环按 log_interval 打 DROBOTICFLOW_PROGRESS (RFC3339 updatedAt)
# 4) norm_stats 缺失时自动补算 (幂等: 存在即跳过, D11); 优先级 上游 NORM_STATS_PATH 显式
# 指定 > 自动计算
# 5) checkpoint 收尾写 model_structure.json (契约快照, 量化/serve/convert 锚点, D13/R-9)
# 与 $OUTPUT_DIR/.artifacts.json
# 6) 训练入口超参: DATASET_PATH (与上游端口二选一上游优先) / CONTRACT_JSON / KEY_MAP /
# RESUME_CKPT / NORM_STATS_PATH / BATCH_SIZE / NUM_TRAIN_STEPS / LR_WARMUP_STEPS /
# LR_PEAK / LR_DECAY_STEPS / SAVE_INTERVAL / KEEP_PERIOD / SEED / NUM_WORKERS / RESUME /
# OVERWRITE / EXP_NAME
from __future__ import annotations
import dataclasses
import functools
import json
import logging
import os
import shutil
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path
import etils.epath as epath # noqa: F401 (openpi 训练组件依赖此命名空间可用)
import flax.nnx as nnx
from flax.training import common_utils
import jax
import jax.numpy as jnp
import numpy as np
import optax
import tqdm_loggable.auto as tqdm
import openpi.models.model as _model
import openpi.shared.normalize as _normalize
import openpi.shared.nnx_utils as nnx_utils
import openpi.training.checkpoints as _checkpoints
import openpi.training.config as _config
import openpi.training.data_loader as _data_loader
import openpi.training.optimizer as _optimizer
import openpi.training.sharding as sharding
import openpi.training.utils as training_utils
import openpi.training.weight_loaders as _weight_loaders
import openpi.transforms as _transforms
from openpi.models.pi0_config import Pi0Config
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as _contract # noqa: E402
from ws_common import input_locate, report_utils # noqa: E402
# 镜像内置官方权重 (构建期下载, D8: 只打 JAX pi05_base 一份)
OFFICIAL_PI05_BASE = "/app/src/openpi_weights/pi05_base/params"
logger = logging.getLogger("openpi")
def _env_int(key: str, default: int) -> int:
raw = os.environ.get(key, "").strip()
if not raw:
return default
try:
return int(raw)
except ValueError:
sys.exit(f"[hyperparam] {key}: invalid int value {raw!r}")
def _env_float(key: str, default: float) -> float:
raw = os.environ.get(key, "").strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
sys.exit(f"[hyperparam] {key}: invalid float value {raw!r}")
# ============================================================
# 契约驱动的 DataConfigFactory
# ============================================================
class ContractDataConfigFactory(_config.DataConfigFactory):
"""把契约 key_map 编译成 repack transforms + 归一化模式的 DataConfig。
多数据集聚合: dataset_paths 为 1..N 个共父目录的数据集根; 所有数据集共用
同一套契约 transforms 与同一份 norm_stats (统计在全量混合上计算), 主
repo_id = 第一个数据集目录名, 其余进 extra_repo_ids (data_loader 侧
ConcatDataset 拼接)。asset_id 单数据集保持目录名 (向后兼容旧统计产物),
多数据集用确定性的混合 id。"""
def __init__(self, dataset_paths: list[Path], contract: _contract.Contract, assets_dir: Path):
if not dataset_paths:
raise ValueError("dataset_paths must be non-empty")
super().__init__(repo_id=dataset_paths[0].name)
self._dataset_paths = dataset_paths
self._contract = contract
self._assets_dir = assets_dir
@property
def asset_id(self) -> str:
names = [p.name for p in self._dataset_paths]
if len(names) == 1:
return names[0]
digest = __import__("hashlib").md5("|".join(names).encode()).hexdigest()[:8]
return f"mix{len(names)}-{digest}"
def create(self, assets_dirs, model_config: _model.BaseModelConfig) -> _config.DataConfig:
del assets_dirs # 数据集与统计均锚定显式路径, 不走全局 assets 目录
c = self._contract
# repack structure: 旧键扁平路径 -> 新键 (image/<slot>, state, actions);
# prompt 键透传 (PromptFromLeRobotTask 在 dataset 层已产出 "prompt",
# repack 重建 dict 时需显式保留, 否则 TokenizePrompt 报 Prompt is required)
image_struct = {slot: src for src, slots in c.vision_map.items() for slot in slots}
state_src = next(k for k in ("state/concat", "state", "observation.state") if k in c.key_map)
action_src = next(k for k in ("action/concat", "action", "actions") if k in c.key_map)
structure = {
"image": image_struct,
"state": state_src,
"actions": action_src,
"prompt": "prompt",
}
use_quantile = c.use_quantile_norm
norm_mode = None if c.norm_mode == "quantile" else c.norm_mode
return dataclasses.replace(
self.base_config or _config.DataConfig(),
repo_id=self._dataset_paths[0].name,
extra_repo_ids=tuple(p.name for p in self._dataset_paths[1:]),
asset_id=self.asset_id,
norm_stats=self._load_norm_stats(),
repack_transforms=_transforms.Group(inputs=[_transforms.RepackTransform(structure)]),
data_transforms=_transforms.Group(inputs=[_transforms.ImagesToHWC(), _transforms.BuildImageMasks()]),
# model_transforms: TokenizePrompt / ResizeImages / PadStatesAndActions
# (缺失时 jax 静默丢语言分支, torch 在 lang_embed 报 None)
model_transforms=_config.ModelTransformFactory()(model_config),
use_quantile_norm=use_quantile,
norm_mode=norm_mode,
action_sequence_keys=(action_src,),
prompt_from_task=True,
)
def dataset_configs(self, model_config: _model.BaseModelConfig) -> list[_config.DataConfig]:
"""每数据集一份 DataConfig (repo_id 各自, extra 为空, 其余共享)。
消费方: norm_stats 统计 (在全量混合上计算共享统计)。"""
base = self.create(None, model_config)
configs = [dataclasses.replace(base, repo_id=p.name, extra_repo_ids=()) for p in self._dataset_paths]
return configs
def _load_norm_stats(self):
stats_dir = self._assets_dir / self.asset_id
try:
stats = _normalize.load(stats_dir)
logging.info(f"[norm_stats] loaded from {stats_dir}")
return stats
except FileNotFoundError:
logging.info(f"[norm_stats] not found in {stats_dir} (will be computed)")
return None
# ============================================================
# TrainConfig 组装
# ============================================================
def resolve_dataset_paths() -> list[Path]:
"""输入根 (上游端口 portId=dataset 优先, DATASET_PATH 兜底) -> 数据集根列表。
根可为单数据集根或多数据集父目录 (expand_dataset_roots 展开); 多数据集
必须共父目录 (HF_LEROBOT_HOME 唯一)。每个数据集根校验 meta/info.json。
平台 workspace (PVC) 输入先整体暂存到节点本地盘再展开 (stage_to_local)。"""
root = input_locate.locate_input_dir(port_id="dataset", env_key="DATASET_PATH")
root = input_locate.stage_to_local(root)
paths = input_locate.expand_dataset_roots(root)
parents = {p.parent for p in paths}
if len(parents) != 1:
sys.exit(f"[input] datasets must share one parent dir for LeRobot local read, got {parents}")
for path in paths:
if not (path / "meta" / "info.json").is_file():
sys.exit(
f"[input] {path} is not a LeRobot v3 dataset root "
"(expected meta/info.json + data/*.parquet); got contents: "
f"{sorted(p.name for p in path.iterdir())[:10]}"
)
return paths
def build_train_config(contract: _contract.Contract, dataset_paths: list[Path], output_dir: Path) -> _config.TrainConfig:
resume_ckpt = os.environ.get("RESUME_CKPT", "").strip()
if resume_ckpt in ("", "official"):
weight_loader: _weight_loaders.WeightLoader
if os.path.isdir(OFFICIAL_PI05_BASE):
weight_loader = _weight_loaders.CheckpointWeightLoader(OFFICIAL_PI05_BASE)
else:
weight_loader = _weight_loaders.CheckpointWeightLoader("gs://openpi-assets/checkpoints/pi05_base/params")
else:
if not os.path.isdir(resume_ckpt):
sys.exit(f"[hyperparam] RESUME_CKPT directory not found: {resume_ckpt}")
weight_loader = _weight_loaders.CheckpointWeightLoader(resume_ckpt)
assets_dir = output_dir / "assets"
exp_name = os.environ.get("EXP_NAME", "").strip() or "pi05_contract"
lr_peak = _env_float("LR_PEAK", 2.5e-5)
# LORA=true: 官方 low_mem LoRA 微调惯例 (config.py pi0_libero_low_mem_finetune 同款):
# LoRA variant + freeze 非 LoRA 参数 (bf16) + 关 EMA。32GB 卡全量微调 init 峰值
# (params+EMA+AdamW 状态 replicated 中间量) 不可行时的冒烟/低显存路径。
use_lora = os.environ.get("LORA", "").strip().lower() in ("1", "true", "yes")
model_kwargs = dict(contract.pi0_config_kwargs())
if use_lora:
model_kwargs["paligemma_variant"] = "gemma_2b_lora"
model_kwargs["action_expert_variant"] = "gemma_300m_lora"
config = _config.TrainConfig(
name="pi05_contract",
project_name="datapipeline",
exp_name=exp_name,
model=Pi0Config(**model_kwargs),
weight_loader=weight_loader,
data=ContractDataConfigFactory(dataset_paths, contract, assets_dir / "pi05_contract"),
assets_base_dir=assets_dir,
# PI_CKPT_LOCAL=1: checkpoint 先写节点本地盘, 训练完成后再整体同步回
# OUTPUT_DIR/checkpoints。平台 workspace 为网络 PVC, orbax/tensorstore 的
# OCDBT manifest 原子写与 zarr .__lock 文件锁在其元数据一致性下会损坏/
# 失败; 本地直写 (本地盘部署) 保持原行为不受影响。
checkpoint_base_dir=(
Path(tempfile.gettempdir()) / "pi-ckpt" / "checkpoints"
if os.environ.get("PI_CKPT_LOCAL", "").strip().lower() in ("1", "true", "yes")
else output_dir / "checkpoints"
),
seed=_env_int("SEED", 0),
batch_size=_env_int("BATCH_SIZE", 8),
num_workers=_env_int("NUM_WORKERS", 4),
num_train_steps=_env_int("NUM_TRAIN_STEPS", 20000),
fsdp_devices=_env_int("FSDP_DEVICES", 1),
log_interval=_env_int("LOG_INTERVAL", 100),
save_interval=_env_int("SAVE_INTERVAL", 1000),
keep_period=_env_int("KEEP_PERIOD", 5000),
overwrite=os.environ.get("OVERWRITE", "").strip().lower() in ("1", "true", "yes"),
resume=os.environ.get("RESUME", "").strip().lower() in ("1", "true", "yes"),
wandb_enabled=False, # 改动点 2: 强制关闭
freeze_filter=Pi0Config(**model_kwargs).get_freeze_filter() if use_lora else nnx.Nothing(),
ema_decay=None if use_lora else _env_float("EMA_DECAY", 0.99),
lr_schedule=_optimizer.CosineDecaySchedule(
warmup_steps=_env_int("LR_WARMUP_STEPS", 1000),
peak_lr=lr_peak,
decay_steps=_env_int("LR_DECAY_STEPS", 30000),
),
optimizer=_optimizer.AdamW(),
)
logging.info(f"[contract] cameras={contract.cameras} "
f"resolution={contract.per_cam_resolution} prefix_tokens={contract.prefix_tokens} "
f"norm_mode={contract.norm_mode}")
return config
# ============================================================
# norm_stats 自动补算 (D11: 幂等, 存在即跳过)
# ============================================================
def compute_norm_stats_if_missing(config: _config.TrainConfig, contract: _contract.Contract) -> None:
explicit = os.environ.get("NORM_STATS_PATH", "").strip()
stats_dir = config.assets_dirs / config.data.asset_id
if explicit:
if not os.path.isfile(os.path.join(explicit, "norm_stats.json")):
sys.exit(f"[norm_stats] NORM_STATS_PATH has no norm_stats.json: {explicit}")
# 显式指定优先: 复制到 assets 目录供 DataConfig 加载
import shutil
shutil.copytree(explicit, stats_dir, dirs_exist_ok=True)
logging.info(f"[norm_stats] copied from explicit NORM_STATS_PATH {explicit} -> {stats_dir}")
return
if (stats_dir / "norm_stats.json").is_file():
logging.info(f"[norm_stats] exists, skip (idempotent): {stats_dir}")
return
# 多数据集: 统计在全量混合上计算 (共享一份 norm_stats, 全部数据集在
# repack+data_transform 空间逐帧累积)
logging.info(f"[norm_stats] computing in repack+data_transform space -> {stats_dir}")
stats = {"state": _normalize.RunningStats(), "actions": _normalize.RunningStats()}
total_frames = 0
for data_config in config.data.dataset_configs(config.model):
dataset = _data_loader.create_torch_dataset(data_config, config.model.action_horizon, config.model)
dataset = _data_loader.TransformedDataset(
dataset,
[
*data_config.repack_transforms.inputs,
*data_config.data_transforms.inputs,
_transforms.InjectDefaultPrompt(None),
],
)
for i in range(len(dataset)):
item = dataset[i]
for key in ("state", "actions"):
stats[key].update(np.asarray(item[key])[None, :] if key == "state" else np.asarray(item[key]))
if (total_frames + i + 1) % 500 == 0:
logging.info(f"[norm_stats] {data_config.repo_id}: {i + 1}/{len(dataset)} frames")
total_frames += len(dataset)
# 多数据集 asset 目录: norm_stats 落在共享 asset_id 下
stats_dir.parent.mkdir(parents=True, exist_ok=True)
_normalize.save(stats_dir, {k: v.get_statistics() for k, v in stats.items()})
logging.info(f"[norm_stats] saved ({total_frames}+ frames): {stats_dir}")
# ============================================================
# 训练主循环 (来源: scripts/train.py main, 改动点见文件头)
# ============================================================
def _emit_progress(stage: str, **fields) -> None:
payload = {"stage": stage, **fields}
payload["updatedAt"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
print("DROBOTICFLOW_PROGRESS " + json.dumps(payload), flush=True)
def _adapt_action_dim(params: dict, action_dim: int) -> dict:
"""D5 / design_v2 §5.2: action_dim resume 适配 — action_in/out_proj kernel 循环 tile。
官方 pi05_base 的 action_dim=32; 契约维 ad != 32 时 (ad<32 截取, ad>32 循环重复)
适配到模型结构。bias/out_features 不受影响; state 走 tokenizer 无投影权重。
"""
for proj, axis in (("action_in_proj", 0), ("action_out_proj", 1)):
node = params.get(proj)
if not isinstance(node, dict) or "kernel" not in node:
continue
kernel = np.asarray(node["kernel"])
src_dim = kernel.shape[axis]
if src_dim == action_dim:
continue
reps = -(-action_dim // src_dim)
tiled = np.tile(kernel, (reps, 1) if axis == 0 else (1, reps))
tiled = tiled[:action_dim] if axis == 0 else tiled[:, :action_dim]
node["kernel"] = tiled
logging.info(f"[resume-adapt] {proj}.kernel {kernel.shape} -> {tiled.shape} (cyclic tile, D5)")
# out_proj 的 bias 维度 = action_dim (in_proj 的 bias 是 out_features 不受影响)
if "bias" in node and np.asarray(node["bias"]).shape[:1] == (src_dim,):
bias = np.asarray(node["bias"])
node["bias"] = np.tile(bias, reps)[:action_dim]
logging.info(f"[resume-adapt] {proj}.bias {bias.shape} -> {node['bias'].shape} (cyclic tile, D5)")
return params
def _load_weights_and_validate(loader, params_shape):
loaded_params = loader.load(params_shape)
import flax.traverse_util as traverse_util
unflattened = traverse_util.unflatten_dict(
{k: v for k, v in traverse_util.flatten_dict(loaded_params).items() if not isinstance(v, jax.ShapeDtypeStruct)}
)
return unflattened
def init_train_state(config, init_rng, mesh, *, resume):
tx = _optimizer.create_optimizer(config.optimizer, config.lr_schedule, weight_decay_mask=None)
def init(rng, partial_params=None):
rng, model_rng = jax.random.split(rng)
model = config.model.create(model_rng)
if partial_params is not None:
graphdef, state = nnx.split(model)
state.replace_by_pure_dict(partial_params)
model = nnx.merge(graphdef, state)
params = nnx.state(model)
params = nnx_utils.state_map(params, config.freeze_filter, lambda p: p.replace(p.value.astype(jnp.bfloat16)))
return training_utils.TrainState(
step=0,
params=params,
model_def=nnx.graphdef(model),
tx=tx,
opt_state=tx.init(params.filter(config.trainable_filter)),
ema_decay=config.ema_decay,
ema_params=None if config.ema_decay is None else params,
)
train_state_shape = jax.eval_shape(init, init_rng)
# DataPipeline 改动: 阈值 4->32 MiB — attention qkv/out proj 等中小权重在
# jax>=0.11 下切分 (即使 out 维) 仍会在 MHA einsum 的 head 维收缩处产生
# 双边 fsdp 歧义; replicate 中小权重, 只切 embedding / gemma expert MLP 等
# 大块 (显存大头), 兼顾收缩安全与省显存
state_sharding = sharding.fsdp_sharding(train_state_shape, mesh, min_size_mbytes=32, log=True)
if resume:
return train_state_shape, state_sharding
partial_params = _load_weights_and_validate(config.weight_loader, train_state_shape.params.to_pure_dict())
partial_params = _adapt_action_dim(partial_params, config.model.action_dim)
# LoRA (freeze_filter 生效): partial 全为冻结权重, init 的 state_map 本就转 bf16;
# 提前在 jit 外转可将 replicated 输入峰值 12GB(fp32) 降至 6GB, 结果 bit 级等效,
# 为 32GB 卡的 FSDP all-gather 留头寸 (D14 单机多卡约束下的显存适配)
if not isinstance(config.freeze_filter, nnx.Nothing):
partial_params = jax.tree.map(
lambda x: x.astype(np.float32).astype(jnp.bfloat16) if np.asarray(x).dtype == np.float32 else x, partial_params
)
replicated_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
train_state = jax.jit(
init,
donate_argnums=(1,),
in_shardings=replicated_sharding,
out_shardings=state_sharding,
)(init_rng, partial_params)
return train_state, state_sharding
def train_step(config, rng, state, batch):
model = nnx.merge(state.model_def, state.params)
model.train()
observation, actions = batch
def loss_fn(model, rng, observation, actions):
chunked_loss = model.compute_loss(rng, observation, actions, train=True)
return jnp.mean(chunked_loss)
train_rng = jax.random.fold_in(rng, state.step)
observation, actions = batch
diff_state = nnx.DiffState(0, config.trainable_filter)
loss, grads = nnx.value_and_grad(loss_fn, argnums=diff_state)(model, train_rng, observation, actions)
params = state.params.filter(config.trainable_filter)
updates, new_opt_state = state.tx.update(grads, state.opt_state, params)
new_params = optax.apply_updates(params, updates)
nnx.update(model, new_params)
new_params = nnx.state(model)
new_state = dataclasses.replace(state, step=state.step + 1, params=new_params, opt_state=new_opt_state)
if state.ema_decay is not None:
new_state = dataclasses.replace(
new_state,
ema_params=jax.tree.map(
lambda old, new: state.ema_decay * old + (1 - state.ema_decay) * new, state.ema_params, new_params
),
)
kernel_params = nnx.state(
model,
nnx.All(
nnx.Param,
nnx.Not(nnx_utils.PathRegex(".*/(bias|scale|pos_embedding|input_embedding)")),
lambda _, x: x.value.ndim > 1,
),
)
info = {
"loss": loss,
"grad_norm": optax.global_norm(grads),
"param_norm": optax.global_norm(kernel_params),
}
return new_state, info
def main(config: _config.TrainConfig, contract: _contract.Contract, output_dir: Path):
logging.info(f"Running on: {jax.devices()}")
# jax>=0.11: 数据只在 batch 轴切 (数据并行度 = device_count / fsdp_devices),
# 整除性按 batch 轴校验, 不再按总设备数 (官方复合轴语义的遗留校验)
data_parallelism = jax.device_count() // max(1, config.fsdp_devices)
if config.batch_size % data_parallelism != 0:
raise ValueError(
f"Batch size {config.batch_size} must be divisible by the data parallelism {data_parallelism} "
f"(devices {jax.device_count()} / fsdp {config.fsdp_devices})."
)
rng = jax.random.key(config.seed)
train_rng, init_rng = jax.random.split(rng)
mesh = sharding.make_mesh(config.fsdp_devices)
data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.data_axis_spec(mesh)))
replicated_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
checkpoint_manager, resuming = _checkpoints.initialize_checkpoint_dir(
config.checkpoint_dir,
keep_period=config.keep_period,
overwrite=config.overwrite,
resume=config.resume,
)
data_loader = _data_loader.create_data_loader(config, sharding=data_sharding, shuffle=True)
data_iter = iter(data_loader)
batch = next(data_iter)
logging.info(f"Initialized data loader:\n{training_utils.array_tree_to_info(batch)}")
train_state, train_state_sharding = init_train_state(config, init_rng, mesh, resume=resuming)
jax.block_until_ready(train_state)
logging.info(f"Initialized train state:\n{training_utils.array_tree_to_info(train_state.params)}")
if resuming:
train_state = _checkpoints.restore_state(checkpoint_manager, train_state, data_loader)
# jax>=0.11: jit in_shardings 必须与实际数组布局一致 (dataloader 已按
# data_sharding 布局产出); 词表 gather 的歧义已由 gemma.encode 的显式
# out_sharding 解决, 不再需要 prompt replicated 特例 (单卡时代遗留)
batch_in_shardings = data_sharding
ptrain_step = jax.jit(
functools.partial(train_step, config),
in_shardings=(replicated_sharding, train_state_sharding, batch_in_shardings),
out_shardings=(train_state_sharding, replicated_sharding),
donate_argnums=(1,),
)
start_step = int(train_state.step)
pbar = tqdm.tqdm(
range(start_step, config.num_train_steps),
initial=start_step,
total=config.num_train_steps,
dynamic_ncols=True,
)
infos = []
for step in pbar:
# jax.set_mesh: jax>=0.11 原生 mesh 上下文 (openpi 的 sharding.set_mesh 是
# 私有 contextvar, jax 侧不可见), 增广 vmap 的 sharding constraint 依赖它
with sharding.set_mesh(mesh), jax.set_mesh(mesh):
train_state, info = ptrain_step(train_rng, train_state, batch)
infos.append(info)
if step % config.log_interval == 0:
stacked_infos = common_utils.stack_forest(infos)
reduced_info = jax.device_get(jax.tree.map(jnp.mean, stacked_infos))
info_str = ", ".join(f"{k}={v:.4f}" for k, v in reduced_info.items())
pbar.write(f"Step {step}: {info_str}")
_emit_progress("train", step=step, maxStep=config.num_train_steps, loss=float(reduced_info["loss"]))
infos = []
batch = next(data_iter)
if (step % config.save_interval == 0 and step > start_step) or step == config.num_train_steps - 1:
_checkpoints.save_state(checkpoint_manager, train_state, data_loader, step)
logging.info("Waiting for checkpoint manager to finish")
checkpoint_manager.wait_until_finished()
# 改动点 5: model_structure.json (契约快照) 写进最终 checkpoint 目录
final_dir = config.checkpoint_dir / str(config.num_train_steps)
if not final_dir.is_dir():
steps = sorted((p.name for p in config.checkpoint_dir.iterdir() if p.name.isdigit()), key=int)
final_dir = config.checkpoint_dir / steps[-1] if steps else config.checkpoint_dir
structure_path = final_dir / "model_structure.json"
structure_path.write_text(contract.to_json(), encoding="utf-8")
logging.info(f"[model_structure] written: {structure_path}")
# PI_CKPT_LOCAL: 本地盘中转的 checkpoint 在结构文件写完后整体同步回产物目录
ckpt_artifact_dir = config.checkpoint_dir
if not config.checkpoint_dir.is_relative_to(output_dir):
ckpt_artifact_dir = output_dir / "checkpoints"
logging.info(f"[ckpt-local] syncing {config.checkpoint_dir} -> {ckpt_artifact_dir}")
shutil.copytree(config.checkpoint_dir, ckpt_artifact_dir, dirs_exist_ok=True)
structure_path = ckpt_artifact_dir / final_dir.name / "model_structure.json"
if not structure_path.exists() and final_dir.name.isdigit():
for p in ckpt_artifact_dir.rglob("model_structure.json"):
structure_path = p
break
report_utils.write_artifacts(
output_dir,
[
{"name": "checkpoint", "type": "file", "portId": "checkpoint", "path": str(ckpt_artifact_dir)},
{"name": "train_report", "type": "json", "portId": "train_report",
"path": str(_write_train_report(output_dir, config, contract, ckpt_artifact_dir / final_dir.name))},
],
)
def _write_train_report(output_dir: Path, config: _config.TrainConfig, contract: _contract.Contract, final_dir: Path):
report = {
"final_step": config.num_train_steps,
"checkpoint_dir": str(final_dir),
"contract": contract.to_dict(),
"derived": contract.to_model_structure()["derived"],
"batch_size": config.batch_size,
"seed": config.seed,
}
path = output_dir / "train_report.json"
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def run() -> None:
logging.basicConfig(level=logging.INFO, force=True, format="%(asctime)s [%(levelname)s] %(message)s")
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
if not output_dir:
sys.exit("[hyperparam] OUTPUT_DIR not set")
contract = _contract.Contract.from_env()
dataset_paths = resolve_dataset_paths()
config = build_train_config(contract, dataset_paths, output_dir)
# LeRobot v3 本地读取: HF_LEROBOT_HOME 指向数据集公共父目录 (多数据集共父,
# 已由 resolve_dataset_paths 校验), repo_id = 目录名。
# lerobot 的该常量在 import 时固化 (utils/constants.py:67) 且 lerobot_dataset.py
# 以 from-import 持有副本, 环境变量 setdefault 对已加载模块无效, 需双处 patch;
# 不 patch 时 root 落到 ~/.cache 导致 load_metadata FileNotFoundError -> 触发
# 联网 get_safe_version 挂死。
lerobot_home = dataset_paths[0].parent
os.environ["HF_LEROBOT_HOME"] = str(lerobot_home)
import lerobot.datasets.lerobot_dataset as _ld
import lerobot.utils.constants as _lc
_lc.HF_LEROBOT_HOME = lerobot_home
_ld.HF_LEROBOT_HOME = lerobot_home
compute_norm_stats_if_missing(config, contract)
main(config, contract, output_dir)
if __name__ == "__main__":
run()

View File

@ -0,0 +1,109 @@
"""torch 训练入口 (design_v2 §5.3, M4)。
与 train_runner 同一契约/超参面, 调 scripts/train_pytorch.py 的 train_loop
(wandb_enabled=False 由 TrainConfig 控制)。torch 官方起点 = j2t 转换产物或
用户外部权重 (PYTORCH_WEIGHT_PATH; 镜像不预置 torch 官方权重, D8)。
checkpoint 结构 (torch): <step>/model.safetensors + optimizer.pt + metadata.pt +
assets/<asset_id>/norm_stats.json; 收尾补写 model_structure.json (契约快照)。
"""
from __future__ import annotations
import importlib.util
import logging
import os
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as _contract # noqa: E402
from pi_pipeline import train_runner # noqa: E402
from ws_common import report_utils # noqa: E402
logger = logging.getLogger("openpi.train_torch")
_TRAIN_TORCH_SCRIPT = Path(__file__).resolve().parents[1] / "openpi" / "scripts" / "train_pytorch.py"
def _load_train_pytorch():
spec = importlib.util.spec_from_file_location("openpi_train_pytorch", _TRAIN_TORCH_SCRIPT)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def main() -> None:
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
if not output_dir:
sys.exit("[hyperparam] OUTPUT_DIR not set")
contract = _contract.Contract.from_env()
dataset_paths = train_runner.resolve_dataset_paths()
# HF_LEROBOT_HOME 双模块 patch (lerobot 常量 import 时固化, 详见 train_runner.run;
# 多数据集共父目录, 已由 resolve_dataset_paths 校验)
lerobot_home = dataset_paths[0].parent
os.environ["HF_LEROBOT_HOME"] = str(lerobot_home)
import lerobot.datasets.lerobot_dataset as _ld
import lerobot.utils.constants as _lc
_lc.HF_LEROBOT_HOME = lerobot_home
_ld.HF_LEROBOT_HOME = lerobot_home
config = train_runner.build_train_config(contract, dataset_paths, output_dir)
# torch 起点权重: PYTORCH_WEIGHT_PATH (j2t 产物或用户外部权重, D8);
# 缺省时由 train_loop 随机初始化 (仅冒烟用)
torch_weights = os.environ.get("PYTORCH_WEIGHT_PATH", "").strip()
if torch_weights:
if not os.path.isfile(os.path.join(torch_weights, "model.safetensors")):
sys.exit(f"[hyperparam] PYTORCH_WEIGHT_PATH has no model.safetensors: {torch_weights}")
import dataclasses
config = dataclasses.replace(config, pytorch_weight_path=torch_weights)
logger.info("[torch] start weights: %s", torch_weights)
else:
logger.warning("[torch] PYTORCH_WEIGHT_PATH empty: random init (smoke only)")
config = dataclasses_replace_precision(config)
# norm_stats 自动补算 (与 JAX 侧同幂等语义)
train_runner.compute_norm_stats_if_missing(config, contract)
train_pytorch = _load_train_pytorch()
train_pytorch.train_loop(config)
# 多进程 DDP (torchrun) 时收尾产物只由 rank0 写 (官方 train_loop 内部保存
# 已有 rank 判断, 这里补齐 model_structure/artifacts 的 rank0 判断)
_is_rank0 = int(os.environ.get("LOCAL_RANK", os.environ.get("RANK", "0"))) == 0
if not _is_rank0:
return
# model_structure.json + artifacts: 找最新 step 目录
steps = sorted((p.name for p in config.checkpoint_dir.iterdir() if p.name.isdigit()), key=int)
final_dir = config.checkpoint_dir / steps[-1] if steps else config.checkpoint_dir
(final_dir / "model_structure.json").write_text(contract.to_json(), encoding="utf-8")
logger.info("[model_structure] written: %s", final_dir / "model_structure.json")
report_utils.write_artifacts(
output_dir,
[
{"name": "checkpoint", "type": "file", "portId": "checkpoint", "path": str(config.checkpoint_dir)},
{"name": "train_report", "type": "json", "portId": "train_report",
"path": str(train_runner._write_train_report(output_dir, config, contract, final_dir))},
],
)
def dataclasses_replace_precision(config):
import dataclasses
precision = os.environ.get("TRAIN_PRECISION", "bfloat16").strip() or "bfloat16"
if precision not in ("bfloat16", "float32"):
sys.exit(f"[hyperparam] TRAIN_PRECISION must be bfloat16 or float32, got {precision!r}")
return dataclasses.replace(config, pytorch_training_precision=precision)
if __name__ == "__main__":
main()

View File

@ -0,0 +1 @@
ccr-29eug8s3-pub.cnc.bj.baidubce.com/public/openpi_bos_fetch:v20260902_192404

View File

@ -0,0 +1,16 @@
# bos_fetch: BOS 前缀 -> 本地数据集目录 (平台数据落地算子)
FROM python:3.11-slim
ARG http_proxy
ARG https_proxy
ARG no_proxy=localhost
ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1
RUN pip install --no-cache-dir bce-python-sdk
WORKDIR /app
COPY --chmod=755 entry/ /app/entry/
COPY --chmod=755 report.sh /app/report.sh
COPY --chmod=a+rX fetch.py /app/fetch.py
COPY --chmod=a+rX ws_common/ /app/ws_common/
# 无 ENTRYPOINT, 平台运行时覆盖

View File

@ -0,0 +1,26 @@
#!/bin/bash
# bos_fetch 镜像构建 (小镜像, 独立 context)
set -euo pipefail
cd "$(dirname "$0")"
PROXY="${http_proxy:-${https_proxy:-http://192.168.16.68:18000}}"
BUILDER="proxybuilder"
if ! docker buildx inspect "$BUILDER" >/dev/null 2>&1; then
docker buildx create --name "$BUILDER" --driver docker-container \
--driver-opt "env.http_proxy=$PROXY" \
--driver-opt "env.https_proxy=$PROXY" \
--driver-opt "env.HTTP_PROXY=$PROXY" \
--driver-opt "env.HTTPS_PROXY=$PROXY" \
--driver-opt "env.no_proxy=localhost"
fi
STAMP="$(date +%Y%m%d_%H%M%S)"
REG="ccr-29eug8s3-pub.cnc.bj.baidubce.com/public"
image="$REG/openpi_bos_fetch:v$STAMP"
docker buildx build --builder "$BUILDER" --load \
--build-arg http_proxy="$PROXY" \
--build-arg https_proxy="$PROXY" \
--build-arg no_proxy="localhost" \
-f Dockerfile -t "$image" .
echo "[build] built: $image"
echo "$image" > .last_image

View File

@ -0,0 +1,13 @@
#!/bin/sh
# bos_fetch 入口: 下载数据集 + 收尾协议
set -eu
echo "[$(date -u +%FT%TZ)] node=${NODE_LABEL:-} run=${RUN_ID:-}"
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
mkdir -p "$OUTPUT_DIR"
rm -rf "$OUTPUT_DIR/dataset"
python /app/fetch.py
sh /app/report.sh

View File

@ -0,0 +1,93 @@
"""bos_fetch: 从 BOS 前缀下载数据集到产物目录 (平台数据落地算子)。
超参/环境:
BOS_URI 必填, bos://bucket/prefix
BOS_ENDPOINT 默认 bj.bcebos.com
BOS_ACCESS_KEY_ID 凭据 (或平台注入的 BOS_ACCESS_KEY_ID / BOS_AK)
BOS_SECRET_ACCESS_KEY
OUTPUT_DIR 平台注入
产物: $OUTPUT_DIR/dataset/<相对路径>, 清单写 $OUTPUT_DIR/.artifacts.json (portId=dataset)。
"""
import json
import os
import sys
from baidubce import protocol as bce_protocol
from baidubce.auth.bce_credentials import BceCredentials
from baidubce.bce_client_configuration import BceClientConfiguration
from baidubce.services.bos.bos_client import BosClient
sys.path.insert(0, "/app")
from ws_common import report_utils # noqa: E402
def main() -> None:
print("[bos_fetch] env keys:", sorted(k for k in os.environ if k.startswith("BOS_")), flush=True)
uri = os.environ.get("BOS_URI", "").strip()
if not uri.startswith("bos://"):
sys.exit(f"[bos_fetch] BOS_URI must be bos://bucket/prefix, got {uri!r}")
rest = uri[len("bos://") :]
bucket, _, prefix = rest.partition("/")
prefix = prefix.strip("/")
# bce-sdk 的 endpoint 只接受 region 形式 (如 bj); 域名形式会报 Invalid endpoint
endpoint = os.environ.get("BOS_ENDPOINT", "bj.bcebos.com").strip() or "bj.bcebos.com"
ak = os.environ.get("BOS_ACCESS_KEY_ID") or os.environ.get("BOS_AK", "")
sk = os.environ.get("BOS_SECRET_ACCESS_KEY") or os.environ.get("BOS_SK", "")
if not ak or not sk:
sys.exit("[bos_fetch] missing BOS credentials (BOS_ACCESS_KEY_ID/BOS_SECRET_ACCESS_KEY)")
output_dir = os.environ.get("OUTPUT_DIR", "").strip()
if not output_dir:
sys.exit("[bos_fetch] OUTPUT_DIR not set")
out = os.path.join(output_dir, "dataset")
# bce-sdk: protocol 须为协议对象 (str 会报 'str' object has no attribute 'name')
client = BosClient(BceClientConfiguration(BceCredentials(ak, sk), endpoint=endpoint, protocol=bce_protocol.HTTPS))
# 目录前缀精确化: BOS prefix 是裸字符串前缀, "robotwin_hammer_head" 会误匹配
# 兄弟目录 "robotwin_hammer_head_front/_wrists"; 追加 "/" 后只匹配该目录内对象。
# 目录列举为空时回退裸前缀 (单文件 URI 语义)。
effective = f"{prefix}/" if prefix and not prefix.endswith("/") else prefix
def list_all(pfx):
marker = ""
keys = []
while True:
resp = client.list_objects(bucket, prefix=pfx if pfx else None, marker=marker, max_keys=1000)
for obj in resp.contents:
if not obj.key.endswith("/"):
keys.append((obj.key, obj.size))
if getattr(resp, "is_truncated", False):
marker = resp.next_marker
else:
return keys
keys = list_all(effective) if effective else list_all(None)
if not keys and effective != prefix:
keys = list_all(prefix)
if not keys:
sys.exit(f"[bos_fetch] no objects under bos://{bucket}/{prefix}")
total = 0
for i, (key, size) in enumerate(keys, 1):
rel = key[len(effective):].lstrip("/") if effective else key
if not rel:
rel = key.rsplit("/", 1)[-1]
dst = os.path.join(out, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
client.get_object_to_file(bucket, key, dst)
total += size
if i % 5 == 0 or i == len(keys):
report_utils.emit_progress("bos_fetch", files=i, bytes=total)
report_utils.write_artifacts(output_dir, [
{"name": "dataset", "type": "directory", "portId": "dataset", "path": "dataset"},
])
print(f"[bos_fetch] done: {len(keys)} files, {total} bytes -> {out}")
if __name__ == "__main__":
main()

View File

@ -0,0 +1,25 @@
#!/bin/sh
# 收尾协议 (算子镜像规范 Part A §A5 / design_v5 §12.2): 三入口共用。
set -eu
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
: "${RESULT_OUTPUT_PATH:?RESULT_OUTPUT_PATH not set}"
: "${STAGE_MANIFEST_PATH:?STAGE_MANIFEST_PATH not set}"
ARTIFACTS='[]'
if [ -f "$OUTPUT_DIR/.artifacts.json" ]; then
ARTIFACTS=$(cat "$OUTPUT_DIR/.artifacts.json")
fi
# JSON 拼装用系统 python3 (仅标准库, 不依赖任何 venv)。
# logs 必须保持空数组, 让平台自动收集 stdout。
ARTIFACTS="$ARTIFACTS" python3 - "$RESULT_OUTPUT_PATH" <<'PY'
import json, os, sys
artifacts = json.loads(os.environ["ARTIFACTS"])
report = {"logs": [], "artifacts": artifacts, "resourceSeries": []}
with open(sys.argv[1], "w") as f:
json.dump(report, f, ensure_ascii=False)
PY
printf '{"dataRefs":[]}' > "$STAGE_MANIFEST_PATH"
echo "DROBOTICFLOW_RESULT_PATH=$RESULT_OUTPUT_PATH"

View File

@ -0,0 +1,6 @@
"""自研共享层 (OpenPi_Dev): 与 RoboTwin2_Dev/ws_common 同源同步副本。
OpenPi 侧当前只消费 input_locate / report_utils 两个模块; camera_map 与
config_override 为 RoboTwin 仿真侧专用, 不在本工程复制。两份 ws_common 的
公共文件修改时需双向同步 (构建上下文各自自包含, 闭环维护)。
"""

View File

@ -0,0 +1,60 @@
"""产物清单与进度标记 (design_v5 §11 report_utils / §12 收尾协议)。
- write_artifacts: 业务收尾时把产物清单 (name/type/portId/path/sizeBytes 实测)
写入 $OUTPUT_DIR/.artifacts.json, report.sh 读取它拼入 RESULT_OUTPUT_PATH 的
artifacts 字段。portId 与注册侧 outputs 端口 id 一致。
- emit_progress: 长任务逐 episode/逐文件打印 DROBOTICFLOW_PROGRESS + JSON
(含 RFC3339 updatedAt), 平台监控页取最后一条。
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from pathlib import Path
def _rfc3339_now() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def emit_progress(stage: str, **fields) -> None:
payload = {"stage": stage}
payload.update(fields)
payload["updatedAt"] = _rfc3339_now()
print("DROBOTICFLOW_PROGRESS " + json.dumps(payload, ensure_ascii=False), flush=True)
def _path_size(path: Path) -> int:
if path.is_file():
return path.stat().st_size
total = 0
for root, _dirs, files in os.walk(path):
for name in files:
try:
total += (Path(root) / name).stat().st_size
except OSError:
pass
return total
def write_artifacts(output_dir, artifacts: list[dict]) -> None:
"""artifacts 每项: {"name","type","portId","path"}; sizeBytes 实测补齐。"""
output_dir = Path(output_dir)
resolved = []
for item in artifacts:
path = Path(item["path"])
size = _path_size(path) if path.exists() else 0
resolved.append(
{
"name": item["name"],
"type": item.get("type", "file"),
"portId": item["portId"],
"path": str(path),
"sizeBytes": size,
}
)
target = output_dir / ".artifacts.json"
target.write_text(json.dumps(resolved, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"[report] artifacts manifest written: {target} ({len(resolved)} items)")

View File

@ -0,0 +1,17 @@
{
"_comment": "RoboTwin 四相机默认契约 (design_v2 §4.5)。CONTRACT_JSON 超参整体注入; 空值时 train_runner 使用等价的内置默认。外部键 = data2lerobot 转换产物键。图像输入固定 1:1 (224x224)。",
"cameras": ["cam1", "cam2", "cam3", "cam4"],
"state": {"dim": 14, "inject": true},
"language": {"enabled": true, "max_token_len": 200},
"action": {"dim": 14, "horizon": 10},
"norm_mode": "min_max",
"key_map": {
"observation.images.cam_high": ["cam1"],
"observation.images.cam_third_view": ["cam2"],
"observation.images.cam_left_wrist": ["cam3"],
"observation.images.cam_right_wrist": ["cam4"],
"observation.state": "state",
"action": "actions",
"task": "prompt"
}
}

59
releases/RELEASES.md Normal file
View File

@ -0,0 +1,59 @@
# OpenPi_Dev 发布记录
每条记录对应一个已通过平台判定层验证的镜像 tag。判定标准与流程见 /home/chao01.wu/DeployAnything/claude_analysis_notes/POC平台算子注册最快实践.md 的"开发迭代闭环"一节。
## da7ce8b-20260908-2 (当前有效)
| 项 | 值 |
|---|---|
| 镜像 | ccr-29eug8s3-pub.cnc.bj.baidubce.com/public/openpi_train_inference:da7ce8b-20260908-2 |
| openpi 副本 | commit da7ce8b (工作区干净) |
| 构建上下文 | manifest_da7ce8b-20260908-2.txt (27 文件 sha256); src-externals/XPolicyLab 由 ../reference/XPolicyLab 构建期拷入; 权重由 scripts/fetch_weights.sh 幂等预取 |
| 平台算子 | 三个算子 version 1.2.0, image 已切本 tag (JAX 训练 operator-9b489829810b7b59db63f30081fa0a07, torch 训练 operator-848270666764d6ce59991129492ed7fa, norm_stats operator-d6ec15d9574813d2c47f3b716ce47dd6) |
| 工作流 | workflow-cf61aefd254f0283327e1f9ec45e7dfa @ project-3990926c0623c46d0f5942ff8febde23 |
本 tag 相对上一有效发布 (v20260902_093240) 的变更:
1. 图像输入固定 1:1: 非 1:1 比例 (4:3/16:9) 与 pos_emb 行切片移除, siglip.py 与 transformers_replace/modeling_siglip.py 还原为官方代码; 契约不再有 aspect_ratio 字段 (旧契约 "1:1" 兼容, 其余非 0 退出)
2. 审计修复转正进镜像: USER=pi 固化 Dockerfile ENV (uid 10001 无 passwd 条目时 torch getuser 崩溃的根治); stdout tee 与失败保活在 entry; 平台运行不再需要任何 containerCommand 注入包装
3. PI_DATA_LOCAL 环境开关移除, 改为平台形态内生行为: $WORKSPACE_ROOT 下的输入根自动暂存到 /tmp/pi-stage/<pid>/ (本地直跑不暂存)
4. 多数据集聚合: 输入根为父目录时展开全部 LeRobot v3 子数据集 (共父目录约束), DataConfig.extra_repo_ids + ConcatDataset 拼接, 共享混合 norm_stats (资产目录 mix<N>-<hash>)
验证证据 (run 详情 operatorImage 均为本 tag):
| case | 结果 | run | 备注 |
|---|---|---|---|
| T1 jax 4cam 默认契约 LORA 100步 | completed | run-f36b9f5e6c586d7675c776748ee4eb05 | 75min |
| T2 torch 4cam FREEZE_BACKBONE 100步 | completed | run-4b7dcfe07f1abd8941024e5ff42bdaf9 | 53min, 零注入直跑 |
| T3 norm_stats 独立算子 | completed | run-1ee7afb2ab6c48206c0d0ad60e868bb1 | 47min |
| T4 jax 1cam | completed | run-0bf63fb5cb2490fc154a89dc935802ca | 26min |
| T5 jax 多数据集 (dataset_test 父目录 4 库, 1cam 契约) | completed | run-165fb50708bd7c47613b3e5381fa6091 | 128min, 需 WF_RUN_TIMEOUT=14400 |
### 重建步骤 (从本仓库状态到平台验证通过)
1. 构建并推送镜像 (默认 docker driver; proxybuilder 容器型 builder 的 registry 请求不走 env 代理且容器创建时固化的代理地址会过期, 不要用):
```bash
cd DataPipeline/OpenPi_Dev && bash scripts/fetch_weights.sh # 幂等
docker buildx build --builder default --push \
--build-arg http_proxy=http://192.168.16.68:18000 \
--build-arg https_proxy=http://192.168.16.68:18000 \
--build-arg no_proxy=localhost \
-f Dockerfile -t ccr-29eug8s3-pub.cnc.bj.baidubce.com/public/openpi_train_inference:<tag> .
```
2. 算子联动: cd .local_dev/platform_test && python3 update_ops.py (读 ~/tmp_daa_reg/cur_tag 为新 tag)
3. 套件复跑: 同目录 IMG_TRAIN=<新tag> python3 platform_run.py cases_t.json; T5 单独 IMG_TRAIN=<新tag> WF_RUN_TIMEOUT=14400 python3 platform_run.py cases_t5.json
4. 判定核对: run 详情 nodeExecutions[].operatorImage 为新 tag 且状态 completed
### 已知事实 (影响复现)
- 平台对节点 K8s job 默认超时约 2h (超时报错 "timed out waiting for kubernetes job ...", 节点 failed); 多数据集统计等长任务在 PUT 工作流体里带 runTimeoutSeconds (platform_run.py 经 WF_RUN_TIMEOUT 环境变量注入)
- 训练/统计的帧级瓶颈是 AV1 软解码 (~2.2s/帧), 与存储无关; 4 库混合 1215x4 帧统计约 178min
- 上游数据集落地用独立小镜像 openpi_bos_fetch:v20260902_192404 (未随本 tag 变更)
## da7ce8b-20260908 (前一脚, 已被 -2 取代)
与 -2 差异: 无输入本地暂存 (PVC 直读)。T1 jax 4cam 通过 (run-08874fed99c9419d7c3698a17d1db711, 59min); 其 T2 因节点调度过慢被主动终止 (run-83845aa1829930c60fc535a9950d8519, cancelled), torch 路线的判定以 -2 的 T2 为准。
## v20260902_093240 (历史)
详见 2026-09-02 的平台矩阵记录 (12 组用例, 其中 torch case 依赖 containerCommand 注入, 属注入伪验证, 已由 -2 的直跑判定取代)。

View File

@ -0,0 +1,28 @@
# 构建上下文清单: openpi_train_inference:da7ce8b-20260908-2
# 生成: 2026-09-09; openpi 副本 commit da7ce8b (git -C openpi rev-parse HEAD)
# 注: build_docker.sh 为编排脚本 (2026-09-09 修正为 default builder + --push), 其内容不进镜像
7d99d2d173caafa2f09e84c71ec8f281ba4281a653ee4f64d778169154ab8363 Dockerfile
d3695251fe5d5667f09168ba61eebeb2d6991cbc7d5253100c0dfd7e24269ac3 build_docker.sh
7f1b36b18f94b35427e1f19dabb3fb133aed98ede5bc1c71c05ae7ec172b3c6d entry/convert.sh
ef3825eca877868f6ff13e9d9af7242db1cd4ae05257001b3ff0aad48559bc6e entry/norm_stats.sh
bd079aeff65bacfb66a630e85c179ee459826e4e1fcc79a300f02eaf6c0fe793 entry/serve_jax.sh
c96a89b9e52cd92f4451ccb416a57c81bec0f824ac4edbab7191b6215a3ee4c1 entry/serve_torch.sh
545e4319dc5719e5f11eb063cc077a491ce49c71e93acae89c332a69932bc343 entry/train_jax.sh
ade1ad9bd40a367e52fe03ed1d29aa609bfdff4ca1691cdc45e135bdba1889a0 entry/train_torch.sh
e0dfc71e7aee27edd8e7fbcb7a8bda694461cf42c4cbb1f1b1decde0a6b00213 pi_pipeline/__init__.py
f0f59071ddeee8218f0cdb05638a99a8998971fc4a254dbc9d65462a17557f9a pi_pipeline/contract.py
e46e4fafefbb28dbc110fe3da334cb5a0b28ba08a2d6d6a5351f0d5384176889 pi_pipeline/convert_runner.py
049d0f1190e12407a641fb5e0e80ec19d8f654de286f6efb27e3e8149d30b2ef pi_pipeline/norm_stats_runner.py
2766cba52cc07b92dfe87417d57e821b2c036852fccd2a993057abf1747a4776 pi_pipeline/serve_xpolicylab.py
402df2ada77395b9f4a3de02a09a6a96521f8cec4c49bf3d2989191217a67c54 pi_pipeline/train_runner.py
3848e8cdba08c5063a6805be8ef832e648d6e6e486bcbc9e0babeaaaf96be33c pi_pipeline/train_torch_runner.py
ada57aeb1eaf023af9654549efd876308f1a6a0928503d35b147e990eb51c120 presets/robotwin_default_contract.json
3d2bf012803d2c54a1ebdef623add05092b1bcaf57c0da9c04958e1eea02bdd7 report.sh
e2fba6e08a30c2cf3bc575f680f1eb8d943b0752539a0c10dc1d8b391445a07c requirements_train.txt
dd180474be035e4e37b4700d51d1b49abf95a846a4e58f8cb8bd702596963e1d scripts/fetch_gcs_public.py
b8a76db38671af9e879b4626a132ad0f56c3151c14696285a220fa3ecb2bd3af scripts/fetch_weights.sh
7c74da027f4bfa8926895e1f11646ff5057f61e00170da1baca4ce8ce073e23d tests/test_contract.py
9655a093fad8fe0bc3bfb81cd3603337cd678e737b767a03005b3c30003eda84 tests/test_letterbox_golden.py
71e0197acdcd5be65221f36b6c6669c482d9ab89e98c06256d7f1e0ef574f847 ws_common/__init__.py
a242279100c882749f8f70c266c2a022aa16f234d83bdc273f61b7ac5eaf0fe8 ws_common/input_locate.py
f8327093bfa7b28d65386f7c4f7574da667090b68e0e6c937bb5093261fdc15f ws_common/report_utils.py

25
report.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/sh
# 收尾协议 (算子镜像规范 Part A §A5 / design_v5 §12.2): 三入口共用。
set -eu
: "${OUTPUT_DIR:?OUTPUT_DIR not set}"
: "${RESULT_OUTPUT_PATH:?RESULT_OUTPUT_PATH not set}"
: "${STAGE_MANIFEST_PATH:?STAGE_MANIFEST_PATH not set}"
ARTIFACTS='[]'
if [ -f "$OUTPUT_DIR/.artifacts.json" ]; then
ARTIFACTS=$(cat "$OUTPUT_DIR/.artifacts.json")
fi
# JSON 拼装用系统 python3 (仅标准库, 不依赖任何 venv)。
# logs 必须保持空数组, 让平台自动收集 stdout。
ARTIFACTS="$ARTIFACTS" python3 - "$RESULT_OUTPUT_PATH" <<'PY'
import json, os, sys
artifacts = json.loads(os.environ["ARTIFACTS"])
report = {"logs": [], "artifacts": artifacts, "resourceSeries": []}
with open(sys.argv[1], "w") as f:
json.dump(report, f, ensure_ascii=False)
PY
printf '{"dataRefs":[]}' > "$STAGE_MANIFEST_PATH"
echo "DROBOTICFLOW_RESULT_PATH=$RESULT_OUTPUT_PATH"

51
requirements_train.txt Normal file
View File

@ -0,0 +1,51 @@
# venv_train: openpi 双栈训练环境 (design_v2 §2 基线 + D9 升级)
# 官方基线 (pyproject): py3.11 / jax[cuda12]==0.5.3 / flax==0.10.2 / torch==2.7.1 / lerobot(v2)
# 本镜像 (D9 升级): py3.12 / jax[cuda12]>=0.7 (sm_120, 5090/5090D) / torch 2.7.1+cu128 /
# lerobot>=0.4.0 (v3, 与 data2lerobot 转换器同大版本线)
# openpi 本体以 --no-deps 安装, 此清单须覆盖其全部运行时依赖 (wandb/gym-aloha 除外:
# wandb 强制关闭, gym-aloha 仅 examples)。
jax[cuda12]>=0.7.0
flax>=0.10.2
orbax-checkpoint
equinox
dm-tree
flatbuffers
fsspec[gcs]
einops
augmax
optax
chex
jaxtyping
beartype
ml_collections
treescope
polars
filelock
typing-extensions
msgpack
msgpack-numpy
numpy>=1.26
opencv-python-headless==4.10.0.84
pillow
pydantic>=2.5
numpydantic
tqdm
tqdm-loggable
tyro
rich
imageio
distrax
gymnasium
matplotlib
# lerobot 钉 0.4.4: v3 API 且 huggingface-hub<0.36 与 transformers==4.53.2 兼容
# (0.5+ 要求 hf-hub>=1.0 与 transformers 4.53.2 冲突); 与 data2lerobot 写侧同版。
lerobot==0.4.4
safetensors
sentencepiece
websockets>=11.0
# serve (XPolicyLab ws 前端) 链: model_server -> utils.load_file 需要 h5py
h5py
transformers==4.53.2
# NCCL 2.29.7 不写入本清单: torch==2.7.1+cu128 钉死 nvidia-nccl-cu12==2.26.x,
# 整体 resolve 会冲突; 由 Dockerfile 在本清单之后以 --no-deps 单独覆盖安装
# (2.27.x 的 ncclAlltoAll 在 sm_120/虚拟化环境报 unhandled system error)

116
scripts/fetch_gcs_public.py Normal file
View File

@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""匿名拉取 GCS 公开桶对象到本地 (无 gsutil 依赖)。
用法: fetch_gcs_public.py <bucket> <prefix> <output_dir> [--required]
走 storage.googleapis.com 公开 JSON API (列表 + 逐对象下载), 经 http_proxy/HTTPS_PROXY
(若设置)。openpi-assets 与 big_vision 桶均公开可匿名读。
"""
from __future__ import annotations
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
from urllib.parse import quote
def _opener() -> urllib.request.OpenerDirector:
proxy = os.environ.get("https_proxy") or os.environ.get("HTTPS_PROXY") or os.environ.get("http_proxy") or os.environ.get("HTTP_PROXY")
handlers = []
if proxy:
# no_proxy 语义按主机粒度; storage.googleapis.com 不在本地网段, 走代理
handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
return urllib.request.build_opener(*handlers)
def list_objects(opener, bucket: str, prefix: str) -> list[str]:
names: list[str] = []
page_token = None
while True:
url = (
f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o"
f"?prefix={quote(prefix, safe='')}&maxResults=1000"
+ (f"&pageToken={page_token}" if page_token else "")
)
with opener.open(url, timeout=60) as resp:
data = json.load(resp)
names.extend(item["name"] for item in data.get("items", []))
page_token = data.get("nextPageToken")
if not page_token:
break
return names
def download(opener, bucket: str, name: str, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
if dest.exists() and dest.stat().st_size > 0:
return
url = f"https://storage.googleapis.com/storage/v1/b/{quote(bucket)}/o/{quote(name, safe='')}?alt=media"
tmp = dest.with_suffix(dest.suffix + ".partial")
last_err: Exception | None = None
for attempt in range(5):
try:
with opener.open(url, timeout=300) as resp, open(tmp, "wb") as f:
while chunk := resp.read(1 << 20):
f.write(chunk)
tmp.rename(dest)
return
except Exception as exc: # noqa: BLE001
last_err = exc
time.sleep(3 * (attempt + 1))
raise RuntimeError(f"download failed: {name}: {last_err}")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("bucket")
parser.add_argument("prefix")
parser.add_argument("output_dir")
parser.add_argument("--required", action="store_true", help="对象列表为空时非 0 退出")
args = parser.parse_args()
opener = _opener()
names: list[str] | None = None
try:
names = list_objects(opener, args.bucket, args.prefix)
except Exception as exc: # noqa: BLE001 某些桶 (big_vision) 禁匿名 list 但对象公开
print(f"[fetch] list failed ({type(exc).__name__}: {exc}); trying direct object GET", flush=True)
out = Path(args.output_dir) / args.prefix
try:
download(opener, args.bucket, args.prefix, out)
except Exception:
# 直读也失败: 若 list 失败是可重试网络错误, 重新抛原始异常语义
if isinstance(exc, urllib.error.HTTPError) and exc.code not in (401, 403):
raise
raise
print(f"[fetch] done: {out}", flush=True)
return
assert names is not None
if not names:
msg = f"no objects under gs://{args.bucket}/{args.prefix}"
if args.required:
sys.exit(f"[fetch] {msg}")
print(f"[fetch] {msg} (skipped)")
return
out_root = Path(args.output_dir)
total = sum(1 for n in names if not n.endswith("/"))
print(f"[fetch] gs://{args.bucket}/{args.prefix}: {total} objects -> {out_root}")
for i, name in enumerate(names, 1):
if name.endswith("/"):
continue
rel = name[len(args.prefix):].lstrip("/")
download(opener, args.bucket, name, out_root / rel)
if i % 50 == 0 or i == total:
print(f"[fetch] {i}/{total}", flush=True)
print(f"[fetch] done: {out_root}")
if __name__ == "__main__":
main()

29
scripts/fetch_openpi.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# 重建 openpi/ 上游源码: 克隆 Physical-Intelligence/openpi @ 基线 commit 并应用本地补丁。
# 补丁来源: openpi 仓库本地提交 da7ce8b (DataPipeline 定制: 1:1-only revert +
# 多数据集 concat), 导出为 patches/openpi_da7ce8b.patch 随本仓库维护。
# 注意: 重建后工作树内容与原 da7ce8b 提交一致, 但 commit hash 不同 (无 .git 历史)。
set -euo pipefail
cd "$(dirname "$0")/.."
BASE="15a9616a00943ada6c20a0f158e3adb39df2ccac"
PATCH="patches/openpi_da7ce8b.patch"
# 幂等: 已存在且 HEAD 为完整仓库则退出
if git -C openpi rev-parse HEAD >/dev/null 2>&1; then
echo "[fetch_openpi] openpi/ 已存在: $(git -C openpi rev-parse --short HEAD)"
exit 0
fi
[ -f "$PATCH" ] || { echo "[fetch_openpi] 缺少补丁文件: $PATCH"; exit 1; }
# GitHub 需走代理
export http_proxy="${http_proxy:-http://192.168.16.68:18000}"
export https_proxy="${https_proxy:-http://192.168.16.68:18000}"
rm -rf openpi
git clone https://github.com/Physical-Intelligence/openpi.git openpi
git -C openpi checkout "$BASE"
git -C openpi apply --check "../$PATCH"
git -C openpi apply "../$PATCH"
echo "[fetch_openpi] done: upstream@$BASE + $PATCH"

35
scripts/fetch_weights.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
# 幂等预取官方权重到 weights_cache/ (供 Dockerfile COPY, 镜像内不再下载)。
# 三层来源: 本地缓存命中 -> jfs 拷贝 -> 代理在线拉取 (GCS 公开桶)。
set -euo pipefail
cd "$(dirname "$0")/.."
CACHE="weights_cache/openpi_weights"
JFS="${OPENPI_WEIGHTS_JFS:-/mnt/data/chao01.wu/OpenPI_Dev/.local_dev/openpi_weights}"
ok_params=0
ok_tok=0
[ -d "$CACHE/pi05_base/params" ] && [ -n "$(ls -A "$CACHE/pi05_base/params" 2>/dev/null)" ] && ok_params=1
[ -s "$CACHE/big_vision/paligemma_tokenizer.model" ] && ok_tok=1
if [ "$ok_params" -eq 1 ] && [ "$ok_tok" -eq 1 ]; then
echo "[fetch_weights] cache OK: $CACHE"
exit 0
fi
mkdir -p "$CACHE/pi05_base" "$CACHE/big_vision"
if [ -d "$JFS/pi05_base/params" ] && [ -s "$JFS/big_vision/paligemma_tokenizer.model" ]; then
echo "[fetch_weights] copy from jfs: $JFS"
[ "$ok_params" -eq 1 ] || cp -a "$JFS/pi05_base/params" "$CACHE/pi05_base/params"
[ "$ok_tok" -eq 1 ] || cp -a "$JFS/big_vision/paligemma_tokenizer.model" "$CACHE/big_vision/"
echo "[fetch_weights] done (from jfs)"
exit 0
fi
echo "[fetch_weights] jfs cache unavailable, fetching online via proxy"
export http_proxy="${http_proxy:-http://192.168.16.68:18000}"
export https_proxy="${https_proxy:-http://192.168.16.68:18000}"
[ "$ok_params" -eq 1 ] || python3 scripts/fetch_gcs_public.py openpi-assets checkpoints/pi05_base/params "$CACHE/pi05_base/params" --required
[ "$ok_tok" -eq 1 ] || python3 scripts/fetch_gcs_public.py big_vision paligemma_tokenizer.model "$CACHE/big_vision" --required
echo "[fetch_weights] done (online)"

View File

@ -0,0 +1,72 @@
---
name: xpolicylab-adapter-check
description: Audit a policy/<POLICY>/ adapter for XPolicyLab standard compliance and PR readiness — file completeness, deploy.yml, Model contract, script conventions, static checks, debug-mode eval, README and checkpoint requirements. Use when asked to check, validate, review, or pre-flight a policy adapter or a policy submission PR in the XPolicyLab repo.
---
# XPolicyLab Adapter Check
Audit `policy/<POLICY>/` against the submission standard and report pass/fail per item with concrete fixes. The full standard lives in `CONTRIBUTING.md` at the repo root — read it before auditing; the list below is the executable summary. `policy/demo_policy/` is the reference implementation.
This is a **read-only audit**: report the fixes, do not apply them unless the user asks. If the user did not name a policy, list the adapters that changed on the current branch and ask which to audit.
## Checks
Run commands from the repo root unless noted.
1. **Files** — required: `README.md`, `__init__.py`, `install.sh`, `eval.sh`, `setup_eval_policy_server.sh`, `setup_eval_env_client.sh`, `deploy.yml`, `deploy.py`, `model.py`. `process_data.sh` / `train.sh` may be absent only for a declared eval-only submission.
2. **deploy.yml** — `policy_name` equals the directory name (the server imports `XPolicyLab.policy.<policy_name>.model`). Ten keys are required and present in every adapter today: `policy_name`, `protocol` (`ws`), `host`, `port`, `bench_name`, `task_name`, `env_cfg_type`, `seed`, `action_type`, `eval_batch`. Keep a key even when a script defaults it — check the whole list, not just `protocol`. `ckpt_name` and `gpu_id` are optional (supplied per run), and model-specific extra keys are fine but must be documented in the policy README.
3. **model.py** — `class Model(ModelTemplate)` implementing `__init__(model_cfg)`, `update_obs`, `update_obs_batch`, `get_action`, `get_action_batch(env_idx_list=None)`, `reset`. `AGENTS.md` at the repo root states the image, path, and dimension rules and why they hold; what to flag when auditing:
- **Dimensions** — a hard-coded action dim, or a private re-implementation of the lookup instead of `get_robot_action_dim_info(env_cfg_type)`. Flag a direct `_robot_info.json` open in `policy/*/model.py` unless it is a documented fallback with a configurable root, as in `policy/AHA_WAM`. Flag a supported robot that is registered in only one of the two robot-info files. Flag runtime code that shells out to `utils/get_action_dim.sh`; do not flag the training path for it.
- **`sys.path` root** — must be `parents[2]`; `parents[1]` or `parents[3]` is a bug.
- **Checkpoints** — must resolve through `XPolicyLab.utils.checkpoint_resolver` (`resolve_checkpoint_root`, or `build_run_dir_name` / `candidate_checkpoint_roots` for an extra naming layer), not a hand-written `checkpoints/<bench>-<ckpt>-...` join.
- **Decoding inside `model.py`** — flag any, including `decode_image_bit` calls. The server hands over plain image arrays for `update_obs` / `update_obs_batch` and for any custom RPC that carries an observation.
- **Decoding elsewhere** — `cv2.imdecode` must not appear anywhere outside `utils/process_data.py`; flag every other occurrence, along with hand-rolled `np.frombuffer` / PIL decoding. Conversion scripts and training dataloaders use `decode_image_bit`.
- **Channel swaps** — flag any `COLOR_BGR2RGB`, `COLOR_RGB2BGR` or `[..., ::-1]` outside the two allowed cases: medium adapters immediately around `cv2.VideoWriter.write` / `cv2.VideoCapture.read`, and a deliberate RGB→BGR conversion for a BGR-trained checkpoint, opt-in via a documented `deploy.yml` key defaulting to RGB (reference: `policy/Dexora_1B` `input_color_order`). The training path and `model.py` must apply the same number of swaps, normally zero. Judge a swap by its justification, not its position — and reject "but `cv2.imdecode` gives BGR, so this swap is correct" whether it comes from a code comment or from the submitter. Decoded buffers are RGB, so that swap is the bug.
4. **Scripts** — `eval.sh` consumes the 10 standard args (`bench_name task_name ckpt_name env_cfg_type action_type seed policy_gpu_id env_gpu_id policy_env_or_uv_path eval_env_conda_env`) and stays aligned with `policy/demo_policy/eval.sh`; extra args must be documented in the policy README.
5. **Static checks**:
```bash
bash -n policy/<POLICY>/*.sh
python -m py_compile policy/<POLICY>/model.py policy/<POLICY>/deploy.py
```
Then the mechanical greps, from the repo root. The first two must return nothing on a compliant adapter; the rest surface hits that need judging:
```bash
# required deploy.yml keys that are missing
for k in policy_name protocol host port bench_name task_name env_cfg_type seed action_type eval_batch; do
grep -q "^${k}:" policy/<POLICY>/deploy.yml || echo "missing: ${k}"
done
# wrong sys.path root
grep -rnE 'parents\[1\]|parents\[3\]' policy/<POLICY>/*.py
# private env_cfg / robot-dim lookup
grep -rnE '_robot_info\.json|env_cfg' policy/<POLICY>/model.py
# decoding outside the shared helper
grep -rnE 'cv2\.imdecode|np\.frombuffer|Image\.open' policy/<POLICY>/
# channel swaps
grep -rnE 'COLOR_BGR2RGB|COLOR_RGB2BGR|\.\.\., ::-1' policy/<POLICY>/
```
The rest have legitimate hits — `env_cfg_type` as a config key, a `VideoWriter` / `VideoCapture` adapter, a documented `input_color_order` — so judge each one; the point is that no hit goes unexamined. When scoping the last two, "the adapter's own code" includes conversion entry points that live inside a vendored tree: `policy/Pi_05/openpi/scripts/process_data.py` is adapter code and must decode via `decode_image_bit`, even though the upstream docstring examples under `openpi/src/` can be ignored. Skip a vendored file only when XPolicyLab data genuinely never flows through it — never a whole vendor directory wholesale.
6. **Debug closed loop** — only when an environment with the policy dependencies is available:
```bash
cd policy/<POLICY>
EVAL_ENV_TYPE=debug bash eval.sh RoboDojo stack_bowls <ckpt_name> arx_x5 joint 0 0 0 <policy_env> base
```
`<policy_env>` (arg 9) follows the adapter's own convention — a conda env name, `uv`, or an environment path; `policy/Pi_05`, for example, requires `uv` (see its README). The trailing `base` is the eval-env conda env (arg 10).
Must reach `[MAIN] eval finished` with no tracebacks. Re-run with `DEBUG_OBS_ENCODED=1` so the debug client sends encoded camera colors and the server-side decode path is exercised. If it cannot be run, report the item as "not run" — never as passed.
7. **Policy README** — install / data / train / eval commands present and consistent with the actual scripts; supported `action_type` / `env_cfg_type`; checkpoint layout; known limitations.
8. **PR readiness** (when auditing a submission PR) — description follows `.github/PULL_REQUEST_TEMPLATE.md`; checkpoint download script included (Hugging Face or ModelScope preferred) if targeting a leaderboard; eval-only status declared with a training-release timeline.
## Report format
- One line per check: ✅ pass / ❌ fail / ⚠️ not run, with a short reason.
- For each ❌: name the file and the exact change, pointing to `policy/demo_policy` where useful.
- End with a verdict: **ready for PR** or **needs fixes** (list blocking items first).
- Then offer to apply the fixes.
Keep the report short — checks that pass need one line, not an explanation. Never mark an item ✅ based on reading code alone when the check is a command you did not run.

View File

@ -0,0 +1,80 @@
---
name: xpolicylab-model-integration
description: Integrate a robot policy into XPolicyLab as a policy/<POLICY>/ adapter. Use when adding or updating a policy adapter, implementing model.py or deploy.yml, wiring install/process_data/train/eval scripts, or debugging debug-mode evaluation in the XPolicyLab repo.
---
# XPolicyLab Model Integration
Add each policy as a self-contained `policy/<POLICY>/` adapter; keep upstream model code unchanged where possible. `policy/demo_policy/` is the minimal reference — mirror it unless the model truly needs more.
## Before starting
Confirm with the user, or state the assumption explicitly in your first reply:
- Where the upstream model code and checkpoints live (repo URL, local path, or "not available yet").
- Which `env_cfg_type` (robot) and `action_type` (`joint` / `ee`) the model supports.
- Whether this is a full integration or eval-only (no `process_data.sh` / `train.sh`).
- Which conda/uv environment to use for the policy side.
## Workflow
1. Read `policy/demo_policy/` (`model.py`, `deploy.py`, `deploy.yml`, `eval.sh`, `README.md`), then the upstream model's inference API, dependencies, and checkpoint layout.
2. Scaffold: `bash scripts/create_policy.sh <POLICY>` (copies `demo_policy`).
3. Implement `model.py` first (contract below). The policy server imports `XPolicyLab.policy.<POLICY>.model`, so keep the directory importable (`__init__.py`). Put environment setup in `install.sh`; add `process_data.sh` / `train.sh` only if the model supports them.
4. Keep `deploy.py` aligned with `policy/demo_policy/deploy.py` unless the environment loop truly differs. Put runtime defaults in `deploy.yml`, keeping the whole key set from `policy/demo_policy/deploy.yml` — `policy_name` (equal to the directory name), `protocol: ws`, `host`, `port`, plus the per-run fields the setup scripts override. Keep a key even where the scripts already default it.
5. Debug without a simulator, from `policy/<POLICY>/`:
```bash
export EVAL_ENV_TYPE=debug
bash eval.sh RoboDojo stack_bowls <ckpt_name> arx_x5 joint 0 0 0 <policy_env> base
```
`<policy_env>` (arg 9) follows the adapter's own convention — a conda env name, `uv`, or an environment path; `policy/Pi_05`, for example, requires `uv` (see its README). The trailing `base` is the eval-env conda env (arg 10).
Fix import, server-startup, action-key, and shape errors until the loop completes. Run once more with `DEBUG_OBS_ENCODED=1` so the debug client sends encoded camera colors and the server-side decode path is exercised.
6. Static checks from the repo root: `bash -n policy/<POLICY>/*.sh` and `python -m py_compile policy/<POLICY>/model.py policy/<POLICY>/deploy.py`.
7. Write `policy/<POLICY>/README.md`: install, data, train, and eval commands, supported `action_type` / `env_cfg_type`, checkpoint layout, known limitations.
## Model contract (`model.py`)
Define `class Model(ModelTemplate)`, importing `ModelTemplate` from `XPolicyLab.model_template`:
| Method | Contract |
| --- | --- |
| `__init__(model_cfg)` | `model_cfg` is `deploy.yml` merged with CLI overrides (`ckpt_name`, `action_type`, `env_cfg_type`, `seed`, `gpu_id`, ...). Load checkpoints and processors here. |
| `update_obs(obs)` / `update_obs_batch(obs_list)` | Store observation dict(s) for the next action call. |
| `get_action()` | Return one action chunk: `list[dict]` of numpy arrays. |
| `get_action_batch(env_idx_list=None)` | Batched chunks aligned with active env indices; fall back to the config batch size when `None`. |
| `reset()` | Clear model state between episodes. |
Action dict keys: dual-arm uses `left_arm_joint_state` / `right_arm_joint_state` plus `left_ee_joint_state` / `right_ee_joint_state`; single-arm drops the `left_` / `right_` prefix; `action_type=ee` replaces `*_arm_joint_state` with `*_ee_pose` as `[x, y, z, qw, qx, qy, qz]`.
Take dimensions from `get_robot_action_dim_info(env_cfg_type)` in `XPolicyLab.utils.process_data`, and register every new robot in **both** robot-info files — `AGENTS.md` at the repo root has the table and explains why the two `get_action_dim` entry points read different files. A private loader is acceptable only when it must run without importing XPolicyLab, and then it needs a configurable root plus a fallback, as `policy/AHA_WAM` does with `env_cfg_root`.
## Conventions
`AGENTS.md` at the repo root states the image, path, and dimension rules that apply to every change; it is always loaded, so follow it rather than re-deriving anything. The three that adapters get wrong most often: `model.py` never decodes (the server already did), offline code decodes only via `decode_image_bit`, and everything is RGB end to end — a `COLOR_BGR2RGB` added after a decode is always a bug, and the only exceptions are medium adapters around `cv2.VideoWriter` / `cv2.VideoCapture` and a documented `deploy.yml` opt-in like `policy/Dexora_1B`'s `input_color_order`.
Adapter-specific conventions on top of those:
- `eval.sh` positional args, same for all adapters: `bench_name task_name ckpt_name env_cfg_type action_type seed policy_gpu_id env_gpu_id policy_env_or_uv_path eval_env_conda_env`.
- **The importable root is `Path(__file__).resolve().parents[2]`** — the parent of the XPolicyLab checkout, since the server imports `XPolicyLab.policy.<POLICY>.model`. `parents[3]` is especially damaging: it silently shadows modules when inserted at `sys.path[0]`.
- **Resolve checkpoints with `XPolicyLab.utils.checkpoint_resolver`**: `resolve_checkpoint_root(model_cfg, checkpoints_dir)` covers the shared precedence (explicit `deploy.yml` path key, `ckpt_name` as a path, the concatenated run-dir name, then `checkpoints/<ckpt_name>`). Adapters with an extra naming layer build on `build_run_dir_name` / `candidate_checkpoint_roots` instead of starting over. Checkpoints land in `checkpoints/<bench_name>-<ckpt_name>-<env_cfg_type>-<action_type>-<seed>/`.
- Observations carry the language prompt under `instruction` (string; fall back to `instructions`). Poses are `[x, y, z, qw, qx, qy, qz]`.
- Trajectory HDF5 files store a singular `instruction` string and camera extrinsics as `extrinsic_matrix`; runtime observations use `extrinsics_matrix`.
- Full observation/trajectory format trees live in the repo README under "Standard Data Formats".
## Debug-mode troubleshooting
| Symptom | Usual cause |
| --- | --- |
| `ModuleNotFoundError` for the policy | Missing `policy/<POLICY>/__init__.py`, or `install.sh` did not install into the env passed as `<policy_env>`. |
| Client retries connect, then gives up | The server crashed during model loading — read the policy-server log, not the client log; model tracebacks are only printed server-side. |
| Action rejected for wrong keys/dims | Action dict keys must match `action_type` and the arm count; take dims from `get_robot_action_dim_info(env_cfg_type)`. |
| `FileNotFoundError` on `env_cfg/<type>.yml` or `_robot_info.json` | The adapter built the `env_cfg` path itself and pointed it inside the repo, or the robot is registered in only one of the two robot-info files. |
| Works plain, fails with `DEBUG_OBS_ENCODED=1` | `model.py` is decoding images itself, or assumes writable arrays — decoded arrays are read-only views, so copy first. |
| Call times out | Inference is slower than `request_timeout_s` (default 120 s); raise it in `deploy.yml`. |
## Reporting back
Finish with: what was created or changed, the exact eval command you ran and its result, supported `action_type` / `env_cfg_type`, expected checkpoint layout, and anything still unverified (e.g. training untested, no real checkpoint available). Never claim the debug loop passed if you could not run it.

View File

@ -0,0 +1 @@
../.agents/skills

View File

@ -0,0 +1 @@
../.agents/skills

View File

@ -0,0 +1,28 @@
<!-- Policy submission PR — see CONTRIBUTING.md for the full standard.
For non-policy changes, delete the sections that do not apply. -->
## Policy
- Name / paper / upstream repo:
- Supported: bench_name=..., env_cfg_type=..., action_type=...
- Training support: full | eval-only (training release ETA: ...)
## Components
- [ ] install.sh
- [ ] model.py (+ __init__.py)
- [ ] images: RGB end to end, decoding only via decode_image_bit, no channel swaps (see CONTRIBUTING.md)
- [ ] deploy.yml (standard key set incl. protocol: ws / host / port, policy_name matches the directory)
- [ ] deploy.py aligned with demo_policy (or divergence explained)
- [ ] eval.sh + setup_eval_policy_server.sh + setup_eval_env_client.sh
- [ ] process_data.sh / train.sh (or eval-only, declared above)
- [ ] policy README with install / data / train / eval commands
## Testing
- [ ] bash -n + py_compile pass
- [ ] EVAL_ENV_TYPE=debug closed loop passes (paste the log tail)
- [ ] Simulator eval: task=..., success=... (if available)
## Checkpoint (required for leaderboard evaluation)
<download script, Hugging Face or ModelScope preferred>
## Limitations / notes
...

20
src-externals/XPolicyLab/.gitignore vendored Normal file
View File

@ -0,0 +1,20 @@
**/__pycache__/**
**/*.egg-info/
.cursor/*
!.cursor/skills
.claude/*
!.claude/skills
**/.venv/**
**/checkpoints/**
train_out/
checkpoints/
runs/
Log/
logs/
# H_RDT local artifacts and large model assets
policy/H_RDT/.cache/
policy/H_RDT/H_RDT/.cache/
policy/H_RDT/H_RDT/t5-v1_1-xxl/
policy/H_RDT/**/logs/
policy/H_RDT/**/wandb/

View File

@ -0,0 +1,71 @@
# XPolicyLab Agent Guide
XPolicyLab wraps each robot policy as a self-contained adapter under `policy/<POLICY>/`. The policy
server imports it as `XPolicyLab.policy.<POLICY>.model`, so this checkout is always used as a
package inside a parent workspace — never as the top-level project.
- Submission standard: [CONTRIBUTING.md](CONTRIBUTING.md). Reference adapter: `policy/demo_policy/`.
Data formats: [README](README.md#-standard-data-formats).
- Two skills carry the end-to-end workflows — `xpolicylab-model-integration` (build an adapter) and
`xpolicylab-adapter-check` (audit one before a PR). They live in `.agents/skills/`, which
`.cursor/skills` and `.claude/skills` symlink to, so Cursor, Claude Code and Codex all load them.
The rules below apply to every change in this repo. Their rationale is in CONTRIBUTING.md.
## Images are RGB from end to end
`decode_image_bit` and `decode_obs_images` return RGB, and the policy server hands `update_obs` /
`update_obs_batch` RGB. Treat this as settled: do not re-derive it from the usual "OpenCV returns
BGR" rule, which does not apply, because XPolicyLab buffers are encoded from RGB arrays and
`cv2.imencode` / `cv2.imdecode` carry channels through JPEG in the order they were given. A
`COLOR_BGR2RGB` added to "fix" a decode is always a bug: it trains on BGR and evaluates on RGB.
No channel conversion belongs in conversion, training, or eval code. Two exceptions only:
- Medium adapters: `COLOR_RGB2BGR` immediately before `cv2.VideoWriter.write(...)`, and
`COLOR_BGR2RGB` immediately after `cv2.VideoCapture.read()`.
- A deliberate RGB→BGR conversion for a checkpoint trained on BGR data, opt-in through a documented
`deploy.yml` key that defaults to RGB (see `input_color_order` in `policy/Dexora_1B`).
## Decoding goes through the shared helpers
`model.py` never decodes. The server decodes every observation it forwards, including for custom
RPCs, so `obs["vision"][<camera>]["color"]` is already an array; adapters only reshape, cast, resize.
Offline code — conversion scripts and training dataloaders — decodes only with `decode_image_bit`
from `XPolicyLab.utils.process_data`. Never hand-roll `cv2.imdecode` / `np.frombuffer` / PIL
decoding: RoboTwin and RoboDojo legacy image-bit layouts are only handled correctly by that
function. Mechanically, `cv2.imdecode` must not appear outside `utils/process_data.py`.
## Paths and dimensions come from the shared helpers
`env_cfg/` lives in the **parent workspace, outside this repo** — `XPolicyLab/env_cfg` does not
exist, so an adapter must never build that path itself.
- **Importable root** in `policy/<POLICY>/model.py` is `Path(__file__).resolve().parents[2]`, the
parent of the checkout. `parents[1]` is the checkout itself and `parents[3]` is unrelated; both
are bugs.
- **Checkpoints** resolve through `XPolicyLab.utils.checkpoint_resolver` — `resolve_checkpoint_root`,
or `build_run_dir_name` / `candidate_checkpoint_roots` when an adapter adds its own naming layer —
never by re-deriving `checkpoints/<bench_name>-<ckpt_name>-<env_cfg_type>-<action_type>-<seed>/`.
- **Action dimensions** come from `get_robot_action_dim_info(env_cfg_type)` in
`XPolicyLab.utils.process_data`, never hard-coded and never through a private re-implementation.
A new robot must be registered in **both** robot-info files, or training and evaluation disagree
about action dimensions:
| File | Read by | Keyed by |
| --- | --- | --- |
| `<parent>/env_cfg/robot/_robot_info.json` | `get_robot_action_dim_info()` and `get_action_dim()` in `XPolicyLab.utils.process_data` — runtime and offline conversion | robot name, from `config.robot` in `<parent>/env_cfg/<env_cfg_type>.yml` |
| `utils/robot/_robot_info.json` | `utils/get_action_dim.sh`, called by `train.sh` | `env_cfg_type` |
The two entry points named `get_action_dim` do **not** read the same file. Runtime and conversion
code imports the Python one; the training path — `train.sh`, or a Python training entry it invokes —
uses `utils/get_action_dim.sh`, which delivers the value as a shell variable, runs on stdlib `json`
alone, and works in a standalone checkout that has no outer `env_cfg/` tree.
## deploy.yml
`policy_name` must equal the directory name. Keep the full key set from
`policy/demo_policy/deploy.yml` — including `protocol: ws`, `host` and `port` — even where the
scripts have a default; per-run fields are overridden at launch.

View File

@ -0,0 +1 @@
@AGENTS.md

View File

@ -0,0 +1,162 @@
# Contributing a Policy to XPolicyLab
This page is the submission standard for `policy/<POLICY>/` adapters: what a complete adapter contains, how to test it, and what a PR must include. For repo-wide concepts and workflows, see the [README](README.md).
Two bundled Agent Skills automate most of this: `xpolicylab-model-integration` builds an adapter, `xpolicylab-adapter-check` audits one before a PR. They live in `.agents/skills/`, which `.cursor/skills` and `.claude/skills` symlink to, so Cursor, Claude Code and Codex all pick them up. [AGENTS.md](AGENTS.md) distills this page into the always-on rules every agent must follow; `CLAUDE.md` just imports it.
## Adapter Standard
### Files
Scaffold with `bash scripts/create_policy.sh <POLICY>` (copies `policy/demo_policy/`), then keep:
```text
policy/<POLICY>/
├── README.md # required: install / data / train / eval guide
├── __init__.py # required: keeps XPolicyLab.policy.<POLICY> importable
├── install.sh # required: policy environment setup
├── eval.sh # required: same-machine evaluation
├── setup_eval_policy_server.sh # required: policy-side server
├── setup_eval_env_client.sh # required: environment-side client
├── deploy.yml # required: runtime config, protocol: ws
├── deploy.py # required: evaluation loop
├── model.py # required: Model adapter class
├── process_data.sh # data conversion (see eval-only exception below)
├── train.sh # training entry (see eval-only exception below)
└── INSTALLATION.md # optional: extra setup notes
```
`process_data.sh` / `train.sh` may be omitted only for an agreed **eval-only** submission: state it in the PR, notify the maintainers ([Contact](README.md#-contact)), and give a timeline for open-sourcing training.
### `model.py`
Define `class Model(ModelTemplate)` (`from XPolicyLab.model_template import ModelTemplate`):
| Method | Contract |
| --- | --- |
| `__init__(model_cfg)` | `model_cfg` is `deploy.yml` merged with per-run overrides (`ckpt_name`, `action_type`, `env_cfg_type`, `seed`, ...). Load checkpoints and processors here. |
| `update_obs(obs)` / `update_obs_batch(obs_list)` | Store observation dict(s) for the next action call. |
| `get_action()` | Return one action chunk: `list[dict]` of numpy arrays. |
| `get_action_batch(env_idx_list=None)` | Batched chunks aligned with active env indices. |
| `reset()` | Clear model state between episodes. |
Action dictionaries use the standard keys (`left_arm_joint_state`, `right_ee_joint_state`, `ee_pose`, ...) with dimensions taken from `get_robot_action_dim_info(env_cfg_type)` in `XPolicyLab.utils.process_data` — never hard-coded, and never through a private re-implementation of the lookup. `env_cfg/` lives in the parent workspace, outside this checkout, so an adapter that assembles that path itself gets it wrong. Observation and trajectory formats: README, [Standard Data Formats](README.md#-standard-data-formats).
A new robot must be registered in **both** robot-info files, or training and evaluation will disagree about action dimensions:
| File | Read by | Keyed by |
| --- | --- | --- |
| `<parent>/env_cfg/robot/_robot_info.json` | `get_robot_action_dim_info()` and `get_action_dim()` in `XPolicyLab.utils.process_data` — runtime and offline conversion | robot name, from `config.robot` in `<parent>/env_cfg/<env_cfg_type>.yml` |
| `utils/robot/_robot_info.json` | `utils/get_action_dim.sh`, called by `train.sh` | `env_cfg_type` |
Two entry points share the name `get_action_dim`, and they do **not** read the same file. Runtime and conversion code imports `get_action_dim` (or `get_robot_action_dim_info`) from `XPolicyLab.utils.process_data`. The training path — `train.sh`, or a Python training entry it invokes — uses `utils/get_action_dim.sh` instead: it delivers the value as a shell variable, runs on stdlib `json` alone (importing the Python module pulls in numpy, cv2, h5py and yaml), and works in a standalone checkout that has no outer `env_cfg/` tree.
Two more shared entry points, so adapters do not re-derive them: the importable root in `policy/<POLICY>/model.py` is `Path(__file__).resolve().parents[2]` (the parent of this checkout — `parents[1]` or `parents[3]` is a bug), and checkpoint directories resolve through `XPolicyLab.utils.checkpoint_resolver` (`resolve_checkpoint_root`, or `build_run_dir_name` / `candidate_checkpoint_roots` when the adapter adds its own naming layer).
`model.py` never decodes images. The policy server decodes every observation it forwards, so `obs["vision"][<camera>]["color"]` is always a plain image array. This holds for `update_obs` / `update_obs_batch` and for any custom RPC a policy exposes that carries an observation, so an adapter with its own deploy loop still must not decode.
In offline code — conversion scripts and training dataloaders that read trajectory files — decode only with `decode_image_bit` from `XPolicyLab.utils.process_data`, never with hand-rolled `cv2.imdecode` / `np.frombuffer` / PIL decoding. RoboTwin/RoboDojo legacy image-bit layouts are only handled correctly by this function.
Images are RGB end to end. `decode_image_bit` returns RGB — treat this as settled and do not re-derive it from the usual "OpenCV returns BGR" rule, which does not apply here: XPolicyLab buffers are encoded from RGB arrays, and `cv2.imencode` / `cv2.imdecode` move channels through JPEG in the order they were given, so the round trip is RGB in, RGB out. No channel conversion belongs in conversion, training, or eval code. Two exceptions: medium adapters — `COLOR_RGB2BGR` immediately before `cv2.VideoWriter.write(...)` and `COLOR_BGR2RGB` immediately after `cv2.VideoCapture.read()` — and a deliberate RGB→BGR conversion for a checkpoint trained on BGR data, which must be opt-in through a documented `deploy.yml` key that defaults to RGB (see `policy/Dexora_1B`'s `input_color_order`). A `cv2.cvtColor(decode_image_bit(...), COLOR_BGR2RGB)` anywhere means training and evaluation disagree on channel order.
### `deploy.yml`
`policy_name` must equal the directory name — the server imports `XPolicyLab.policy.<policy_name>.model`, and the setup scripts derive the name from the directory. Keep `protocol: ws` (`legacy_tcp` is for unmigrated legacy adapters only). Per-run fields are overridden by the setup scripts; put stable defaults here. Reference (`policy/demo_policy/deploy.yml`):
```yaml
policy_name: demo_policy
protocol: ws
host: localhost
port: null
bench_name: null
task_name: null
ckpt_name: null
env_cfg_type: null
seed: null
action_type: null
gpu_id: null
eval_batch: false
```
All of these are required except `ckpt_name` and `gpu_id`, which the setup scripts supply per run. Keep a key even when a script already defaults it, and document model-specific extra keys in the policy README.
### Scripts
All adapters share the same entry-point conventions (argument meanings: README, [Common Workflow](README.md#-common-workflow)):
```bash
bash eval.sh <bench_name> <task_name> <ckpt_name> <env_cfg_type> <action_type> <seed> \
<policy_gpu_id> <env_gpu_id> <policy_env_or_uv_path> <eval_env_conda_env>
```
`eval.sh` starts the policy server, waits for it, runs the environment client, and cleans up — keep it aligned with `policy/demo_policy/eval.sh`; document any extra arguments in the policy README. Checkpoints resolve to `checkpoints/<bench_name>-<ckpt_name>-<env_cfg_type>-<action_type>-<seed>/` unless the policy README documents another layout.
### Policy README
All policy READMEs share one template — [policy/demo_policy/README.md](policy/demo_policy/README.md) is the minimal reference, [policy/AHA_WAM/README.md](policy/AHA_WAM/README.md) shows a complex adapter (model assets, custom environment variables):
1. **Header**: `**Contributor:** ... | **Paper:** ... | **arXiv:** ... | **Original code:** ...`, then 1–3 sentences on the model and what the adapter supports; mention vendored upstream directories here.
2. **Pointer paragraph** (verbatim): shared conventions link to the root README, official results link to the [RoboDojo LeaderBoard](https://robodojo-benchmark.com/LeaderBoard).
3. **Sections in order**: `Installation`, `Data Processing`, `Training`, `Evaluation`, plus `Model Assets` / `Configuration` / `Notes` only when the adapter needs them. State explicitly when a stage is unsupported (eval-only, upstream-native data, ...).
4. **Policy-specific content only**: one command template plus one runnable example per stage, extra arguments, required environment variables, `deploy.yml` keys, checkpoint-layout deviations. Do not restate shared argument tables or the split-machine flow — link to the root README instead.
## Testing
Run these in order before opening a PR.
**1. Static checks** (repo root):
```bash
bash -n policy/<POLICY>/*.sh
python -m py_compile policy/<POLICY>/model.py policy/<POLICY>/deploy.py
```
**2. Debug closed loop** — no simulator needed; verifies imports, server startup, observation serialization, action keys and dimensions, and batch logic:
```bash
cd policy/<POLICY>
export EVAL_ENV_TYPE=debug
bash eval.sh RoboDojo stack_bowls <ckpt_name> arx_x5 joint 0 0 0 <policy_env> base
```
The run must reach `[MAIN] eval finished` with no tracebacks.
**3. Simulator evaluation** — recommended for every PR and required before a leaderboard entry is published: run the same `eval.sh` with `EVAL_ENV_TYPE=sim` (or unset) inside a RoboDojo / RoboTwin workspace and record task success rates.
## PR Standard
Title: `[policy] <POLICY>: <short summary>`, e.g. `[policy] FastWAM: add RoboDojo adapter`.
Every submission PR must include the adapter following the standard above, a working policy README, and test evidence. For **official leaderboard evaluation** ([RoboDojo](https://robodojo-benchmark.com/LeaderBoard), [RoboTwin](https://robotwin-platform.github.io/leaderboard)), the PR description must also carry a checkpoint download script — Hugging Face or ModelScope preferred — so we can reproduce your results; we evaluate and publish entries as soon as possible.
### PR description template
GitHub pre-fills this from [.github/PULL_REQUEST_TEMPLATE.md](.github/PULL_REQUEST_TEMPLATE.md):
```markdown
## Policy
- Name / paper / upstream repo:
- Supported: bench_name=..., env_cfg_type=..., action_type=...
- Training support: full | eval-only (training release ETA: ...)
## Components
- [ ] install.sh
- [ ] model.py (+ __init__.py)
- [ ] images: RGB end to end, decoding only via decode_image_bit, no channel swaps (see CONTRIBUTING.md)
- [ ] deploy.yml (standard key set incl. protocol: ws / host / port, policy_name matches the directory)
- [ ] deploy.py aligned with demo_policy (or divergence explained)
- [ ] eval.sh + setup_eval_policy_server.sh + setup_eval_env_client.sh
- [ ] process_data.sh / train.sh (or eval-only, declared above)
- [ ] policy README with install / data / train / eval commands
## Testing
- [ ] bash -n + py_compile pass
- [ ] EVAL_ENV_TYPE=debug closed loop passes (paste the log tail)
- [ ] Simulator eval: task=..., success=... (if available)
## Checkpoint (required for leaderboard evaluation)
<download script, Hugging Face or ModelScope preferred>
## Limitations / notes
...
```

View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -0,0 +1,485 @@
<div align="center">
<h1>XPolicyLab</h1>
<p><strong>A Unified Standard and Open Ecosystem for Robot Policy Evaluation and Deployment</strong></p>
<p>
<a href="https://xpolicylab.github.io/">Website</a> |
<a href="https://arxiv.org/abs/2608.09892">arXiv</a> |
<a href="https://github.com/XPolicyLab/XPolicyLab">GitHub</a> |
<a href="https://robodojo-benchmark.com/LeaderBoard">RoboDojo Leaderboard</a> |
<a href="https://robotwin-platform.github.io/leaderboard">RoboTwin Leaderboard</a>
</p>
<img src="assets/teaser.png" alt="XPolicyLab overview" width="100%"/>
<p><em>Connecting N policies to M evaluation environments — from O(N×M) down to O(N+M).</em></p>
</div>
XPolicyLab is the shared layer between policy code and evaluation environments. Keep each model's dependencies, checkpoints, and training recipes under `policy/<POLICY>/`; XPolicyLab handles the parts that are boring but easy to get wrong — serving, observation/action contracts, and eval wiring. As of August 2026, the ecosystem integrates **41 robot policies** spanning VLA, world-action, imitation-learning, and memory-augmented families, and the same adapters serve RoboTwin, RoboDojo simulation, and standardized real-robot evaluation.
Start here for repo-level concepts and integration steps. For install commands, checkpoint layout, and training details, jump to that policy's README — it is the source of truth for its model.
## 📚 Contents
- [What XPolicyLab Enables](#-what-xpolicylab-enables)
- [Supported Benchmarks And Infrastructure](#-supported-benchmarks-and-infrastructure)
- [Integrated Policies](#-integrated-policies)
- [Framework Overview](#-framework-overview)
- [Quick Start](#-quick-start)
- [Common Workflow](#-common-workflow)
- [Deployment Flow](#-deployment-flow)
- [Standard Data Formats](#-standard-data-formats)
- [Data And Checkpoints](#-data-and-checkpoints)
- [Add Your Own Policy](#-add-your-own-policy)
- [Citation](#-citation)
- [Contact](#-contact)
## 🚀 What XPolicyLab Enables
- **Environment isolation**: run the policy model in its own conda/uv environment while the simulator, benchmark, or robot client runs separately.
- **Remote deployment**: connect the policy server and environment client through websocket, either on one machine or across machines.
- **A common adapter contract**: use the same high-level lifecycle for installation, data conversion, training, serving, and evaluation.
- **A large policy zoo**: reuse adapters for VLA/WAM policies, imitation-learning baselines, and reference templates.
- **Benchmark and infra integration**: mount XPolicyLab into benchmark or simulator workspaces without coupling policy code to one environment.
## 🌐 Supported Benchmarks And Infrastructure
XPolicyLab is benchmark-agnostic: any benchmark, simulator, or real-robot setup can plug in as an environment client against the same policy-side interface — one adapter per policy, one client per environment. Two public benchmarks are already integrated, and their official leaderboards are powered by XPolicyLab submissions.
<div align="center">
<img src="assets/benchmarks.png" alt="Cross-platform evaluation through XPolicyLab" width="70%"/>
<p><em>Cross-platform evaluation through a shared codebase and standardized serving interface.</em></p>
</div>
**Benchmarks**
- **[RoboDojo](https://github.com/RoboDojo-Benchmark/RoboDojo)**: simulator-backed evaluation and RoboDojo-format data exports. The [RoboDojo Leaderboard](https://robodojo-benchmark.com/LeaderBoard) covers 42 simulation tasks across five capability dimensions (Generalization, Precision, Long-Horizon, Memory, Open) plus 18 real-robot tasks on three bimanual embodiments.
- **[RoboTwin](https://github.com/RoboTwin-Platform/RoboTwin)**: benchmark and data source through policy-specific adapters and conversion scripts. The [RoboTwin 2.0 Leaderboard](https://robotwin-platform.github.io/leaderboard) covers bimanual manipulation across 50 tasks under clean and randomized settings.
**Infrastructure**
- **[RLinf](https://github.com/RLinf/RLinf)** *(coming soon)*: infrastructure target for policy development and deployment workflows.
- **StarVLA**: infrastructure and policy stack; see [policy/starVLA](policy/starVLA/README.md).
## 🧭 Integrated Policies
41 policies are currently integrated, spanning VLA, world-action, imitation-learning, and memory-augmented families. Top-level adapters live in `policy/`; each policy README documents that model's paper/repo link, environment, data format, training entrypoint, and checkpoint layout.
| [A1](policy/A1/README.md) | [AHA-WAM](policy/AHA_WAM/README.md) | [ABot-M0](policy/Abot_M0/README.md) | [Being-H05](policy/Being_H05/README.md) | [DM0](policy/Dexbotic_DM0/README.md) | [Dexora-1B](policy/Dexora_1B/README.md) |
|:---:|:---:|:---:|:---:|:---:|:---:|
| [DreamZero](policy/DreamZero/README.md) | [EventVLA](policy/EventVLA/README.md) | [FastWAM](policy/FastWAM/README.md) | [G0](policy/GalaxeaVLA/README.md) | [G0.5](policy/G05/README.md) | [GO-1](policy/GO1/README.md) |
| [GR00T-N1.7](policy/GR00T_N17/README.md) | [GigaWorld-Policy](policy/GigaWorldPolicy/README.md) | [H-RDT](policy/H_RDT/README.md) | [Hy-Embodied-0.5-VLA](policy/Hy_Embodied_05_VLA/README.md) | [InternVLA-A1](policy/InternVLA_A1/README.md) | [InternVLA-A1.5](policy/InternVLA_A1_5/README.md) |
| [LDA-1B](policy/LDA_1B/README.md) | [LingBot-VA](policy/LingBot_VA/README.md) | [LingBot-VLA](policy/LingBot_VLA/README.md) | [Mem-0](policy/Mem_0/README.md) | [MolmoAct2](policy/MolmoACT2/README.md) | [OpenVLA-OFT](policy/OpenVLA_OFT/README.md) |
| [π0](policy/Pi_0/README.md) | [π0.5](policy/Pi_05/README.md) | [π0-Fast](policy/Pi_0_Fast/README.md) | [RDT-1B](policy/RDT_1B/README.md) | [RISE](policy/RISE/README.md) | [SmolVLA](policy/SmolVLA/README.md) |
| [Spatial Forcing](policy/Spatial_Forcing/README.md) | [Spirit v1.5](policy/Spirit_v15/README.md) | [TinyVLA](policy/TinyVLA/README.md) | [X-VLA](policy/X_VLA/README.md) | [X-WAM](policy/X_WAM/README.md) | [Xiaomi-Robotics-0](policy/Xiaomi_Robotics_0/README.md) |
| [Xiaomi-Robotics-1 (XR-1)](policy/Xiaomi_Robotics_1/README.md) | [StarVLA](policy/starVLA/README.md) | [ACT](policy/ACT/README.md) | [DP](policy/DP/README.md) | [demo_policy](policy/demo_policy/README.md) | |
Adding a policy of your own, or entering a leaderboard, both go through a PR — see [Add Your Own Policy](#-add-your-own-policy).
## 🧩 Framework Overview
XPolicyLab separates model-side dependencies from environment-side dependencies, so each side retains its native stack and may run locally or remotely. One adapter serves benchmarks, simulators, and physical robots.
<div align="center">
<img src="assets/infra.png" alt="XPolicyLab infrastructure" width="100%"/>
<p><em>Infrastructure of XPolicyLab. One adapter serves benchmarks, simulators, and physical robots.</em></p>
</div>
```text
Policy environment Evaluation / benchmark environment
------------------ ----------------------------------
policy/<POLICY>/model.py <---ws---> env client / simulator / robot
policy server environment client
deploy.yml runtime config benchmark task and observation API
```
A typical adapter contains:
```text
policy/<POLICY>/
├── README.md # policy-specific guide
├── INSTALLATION.md # optional detailed setup notes
├── __init__.py # keeps XPolicyLab.policy.<POLICY> importable
├── install.sh # environment setup
├── process_data.sh # optional data conversion
├── train.sh # optional training
├── eval.sh # same-machine evaluation
├── setup_eval_policy_server.sh # policy-side server
├── setup_eval_env_client.sh # environment-side client
├── deploy.yml # runtime config
├── deploy.py # evaluation loop
└── model.py # model adapter
```
`model.py` implements the model-facing API. `deploy.py` bridges environment observations to model-server calls. Use [policy/demo_policy](policy/demo_policy/README.md) as the minimal adapter reference.
`model.py` should define a `Model` class with this shape:
| Method | Contract |
| --- | --- |
| `__init__(model_cfg)` | Load model config, checkpoints, processors, and runtime overrides from `deploy.yml`. |
| `update_obs(obs)` | Update model state from one observation dictionary. |
| `update_obs_batch(obs_list)` | Update model state from a list of observation dictionaries. |
| `get_action()` | Return one action chunk as a list of action dictionaries. |
| `get_action_batch(env_idx_list=None)` | Return batched action chunks aligned with active environment indices. |
| `reset()` | Clear model-side state between evaluation episodes. It takes no arguments — a policy that needs a first observation should `reset()` and then take a normal `update_obs`. |
The policy server decodes camera colors before `update_obs` / `update_obs_batch`, so `obs["vision"][<camera>]["color"]` always arrives as an image array — `model.py` never decodes.
The default policy-server protocol is websocket (`protocol: ws` in `deploy.yml`); `legacy_tcp` exists only for adapters that have not migrated yet. The transport handles reconnects, retries, keepalive, and long model-loading cold starts for you — a normal adapter never touches it.
<details>
<summary>Transport details and timeout tuning (only if evaluation hangs or drops)</summary>
- **Retries are safe**: each request carries a `request_id` that the client reuses across reconnects, and the server answers duplicates from a cache instead of running a non-idempotent call twice. A `timeout` error is the exception — the server may still be running the call, so treat it as fatal for that trial rather than retrying.
- **Server restarts abort the run**: if a reconnect lands on a different server process, the client raises `ServerRestartedError`, because the fresh server lost the model state.
- **Cold start**: the server loads the model before opening its port, so an early client just retries (default budget 15 min). `eval.sh` also gates the client behind `wait_for_policy_server.sh`.
- **Errors**: the client only sees `str(exc)`; the full traceback of a model failure is logged on the *policy server* side, so look there first.
- **Serialization** is msgpack with numpy support (`torch.Tensor` auto-converts). Three quirks: `tuple` arrives as `list`, decoded numpy arrays are read-only views (copy before in-place edits), and int dict keys arrive as strings.
Optional `deploy.yml` keys — omit them to keep the defaults:
| Key | Default | Purpose |
| --- | --- | --- |
| `request_timeout_s` | `120.0` | Timeout for one `update_obs` / `get_action` call — raise it for slow inference. |
| `max_connect_attempts` | `180` | Cold-start retries while the server is still loading. |
| `connect_retry_delay_s` | `5.0` | Delay between those retries. |
| `max_connect_seconds` | `900.0` | Wall-clock cap on the whole retry loop; `0` disables it. |
| `connect_timeout_s` | `30.0` | Timeout for one connect attempt. |
| `handshake_timeout_s` | `60.0` | Timeout for the HELLO round-trip. |
| `ws_ping_interval_s` / `ws_ping_timeout_s` | `20.0` | Keepalive ping/pong; `null` disables. |
| `close_timeout_s` | `10.0` | Cap on the closing handshake. |
</details>
## ⚡ Quick Start
Clone XPolicyLab as a normal Python project for adapter development, offline checks, training from prepared data, or your own environment client:
```bash
mkdir demo_env
cd demo_env
git clone https://github.com/XPolicyLab/XPolicyLab.git
cd XPolicyLab
pip install -e .
```
You do not need a simulator to start model-side development: the bundled downloader fetches prepared RoboDojo data — several simulator export versions plus HDF5 `RoboDojo_real` real-world data — for training and offline debugging. If you use `XPolicyLab/` as a subpackage inside the RoboDojo repository, follow RoboDojo's own data download scripts instead.
Download a small Hugging Face demo bundle and keep the data next to `XPolicyLab/`:
```bash
# From demo_env/XPolicyLab
bash scripts/RoboDojo/download_robodojo_data.sh demo
```
This creates:
```text
demo_env/
├── data/ # demo data, including a small 10-episode HuggingFace bundle
└── XPolicyLab/
```
The same script pulls the full exports — `hdf5`, `lerobot_v3.0`, `lerobot_v2.1`, and `real` (real-world HDF5) — each into its own `../data/` folder.
With this setup, you can test data conversion, model loading, training scripts, and debug-mode evaluation before connecting to a simulator-backed benchmark.
```bash
export EVAL_ENV_TYPE=debug
cd policy/demo_policy
bash install.sh
bash eval.sh RoboDojo stack_bowls demo arx_x5 joint 0 0 0 base base
```
The template for any adapter is the same — swap `demo_policy` and the argument values:
```bash
export EVAL_ENV_TYPE=debug
cd policy/<POLICY>
bash eval.sh <bench_name> <task_name> <ckpt_name> <env_cfg_type> <action_type> \
<seed> <policy_gpu_id> <env_gpu_id> <policy_env_or_uv_path> <eval_env_conda_env>
```
For RoboDojo simulation, mount `XPolicyLab/` beside the simulator-side `env_cfg/`, `scripts/`, `src/eval_client/`, and `task/` directories.
## 🔄 Common Workflow
Most adapters expose the same top-level shape. Some policies add extra arguments, consume upstream-native datasets, or skip training support. Follow the policy README when it differs from this template.
```bash
cd policy/<POLICY>
# Install the policy runtime.
bash install.sh
# Optional: convert or prepare policy-specific data.
bash process_data.sh <bench_name> <ckpt_name> <env_cfg_type> <action_type> [extra_args...]
# Optional: train.
bash train.sh <bench_name> <ckpt_name> <env_cfg_type> <action_type> <seed> <gpu_id> [extra_args...]
# Evaluate on one machine.
bash eval.sh <bench_name> <task_name> <ckpt_name> <env_cfg_type> <action_type> <seed> \
<policy_gpu_id> <env_gpu_id> <policy_env_or_uv_path> <eval_env_conda_env>
```
### What the arguments mean
When you run `eval.sh`, you are mostly answering: **which benchmark family**, **which task to run now**, **which checkpoint to load**, **which robot setup**, **joint or end-effector actions**, and **which seed**. The same names travel through `process_data.sh`, `train.sh`, and `eval.sh`, so you do not have to rename things at every step.
| Argument | In plain English | Examples |
| --- | --- | --- |
| `bench_name` | Which benchmark or dataset family this run belongs to | `RoboDojo`, `RoboTwin` |
| `task_name` | The task the environment client should run right now | `stack_bowls`, `push_T` — can differ from the tasks seen during training |
| `ckpt_name` | Which weights to load: a short run nickname, the full run folder name, or a path | `cotrain`, `RoboDojo-cotrain-arx_x5-joint-0`, `checkpoints/my_run/` |
| `env_cfg_type` | Robot / camera / scene configuration key | `arx_x5` |
| `action_type` | Action space the policy outputs | usually `joint` or `ee` |
| `seed` | Training or evaluation seed / layout id | `0`, `1`, `2` |
| `policy_gpu_id` / `env_gpu_id` | Which GPU runs the model vs. the simulator/client | `0`, `1` |
| `policy_env_or_uv_path` | Conda env name or uv env path for the policy server | your policy-side env |
| `eval_env_conda_env` | Conda env for the simulator / robot client | your eval-side env |
**How `ckpt_name` resolves.** Usually you pass the short nickname used during training, such as `cotrain`, and XPolicyLab combines it with the other args into `checkpoints/RoboDojo-cotrain-arx_x5-joint-0/`. You can also pass the full folder name, or a path — relative paths resolve from the policy directory, absolute paths work too. Some adapters honor explicit keys in `deploy.yml` (`checkpoint_path`, `model_path`, ...). When in doubt, check the policy README.
**A concrete eval example:**
```bash
cd policy/AHA_WAM
bash eval.sh RoboDojo stack_bowls cotrain arx_x5 joint 0 0 0 aha_wam robodojo
# loads checkpoints/RoboDojo-cotrain-arx_x5-joint-0/ and evaluates on stack_bowls
```
## 🔌 Deployment Flow
During evaluation, the policy server and the environment client talk over websocket. That split is what lets you keep Isaac Sim / robot drivers on one machine and a heavy VLA on another.
For same-machine evaluation, `eval.sh` is enough — it starts the server, runs the client, and cleans up when you are done.
For split-machine deployment, start the policy server on the GPU machine and bind to `0.0.0.0` so other machines can reach it. The client connects to the policy machine's real IP, not `0.0.0.0`.
```bash
cd policy/<POLICY>
bash setup_eval_policy_server.sh \
<bench_name> <task_name> <ckpt_name> <env_cfg_type> <action_type> <seed> \
<policy_gpu_id> <policy_env_or_uv_path> <policy_server_port> 0.0.0.0
```
Then start the environment client on the simulator or robot machine:
```bash
cd policy/<POLICY>
bash setup_eval_env_client.sh \
<bench_name> <task_name> <ckpt_name> <env_cfg_type> <action_type> <seed> \
<env_gpu_id> <eval_env_conda_env> <additional_info> \
<policy_server_port> <policy_server_ip>
```
`<additional_info>` is a comma-separated `key=value` string forwarded to the environment client. `eval.sh` builds it automatically as `ckpt_name=<ckpt_name>,action_type=<action_type>`, which is the right default for most adapters.
`EVAL_ENV_TYPE` selects the environment-side backend:
- unset or `sim`: real simulator-backed evaluation, when the integration is installed.
- `debug`: offline wiring check — no Isaac, no robot, just shapes and IO.
- `real`: real-robot client path, where the hardware integration exists.
## 📐 Standard Data Formats
XPolicyLab standardizes the observation and trajectory dictionaries passed between adapters, converters, and environment clients. Individual policies may convert this standard format into their upstream-native format.
All pose values use `[x, y, z, qw, qx, qy, qz]`. Images are RGB end to end — stored image bits are encoded from RGB frames, and no channel conversion happens anywhere in the pipeline. Note one naming quirk: runtime observations carry camera extrinsics as `extrinsics_matrix`, while trajectory files store `extrinsic_matrix`.
<details>
<summary>Observation Data Format</summary>
```text
Observation Data Format
├── data_format_version string, optional
├── instruction / instructions string or list[str]
├── env_idx int, optional for batched eval
├── additional_info/
│ └── frequency int, optional
├── vision/
│ ├── cam_head/
│ │ ├── color (H, W, 3) RGB, decoded by the server
│ │ ├── depth (H, W) or (H, W, 1), optional
│ │ ├── intrinsic_matrix (3, 3), optional
│ │ ├── extrinsics_matrix (4, 4), optional
│ │ └── shape (2,) or (3,), optional
│ ├── cam_left_wrist/ optional
│ ├── cam_right_wrist/ optional
│ ├── cam_wrist/ optional for single-arm robots
│ └── cam_third_view/ optional
└── state/
├── left_arm_joint_state (DOF,), optional
├── left_ee_joint_state (EEF_DOF,), optional
├── left_ee_pose (7,), optional
├── left_tcp_pose (7,), optional
├── left_delta_ee_pose (7,), optional
├── right_arm_joint_state (DOF,), optional
├── right_ee_joint_state (EEF_DOF,), optional
├── right_ee_pose (7,), optional
├── right_tcp_pose (7,), optional
├── right_delta_ee_pose (7,), optional
├── arm_joint_state (DOF,), optional for single-arm robots
├── ee_joint_state (EEF_DOF,), optional for single-arm robots
├── ee_pose (7,), optional for single-arm robots
├── tcp_pose (7,), optional for single-arm robots
├── delta_ee_pose (7,), optional for single-arm robots
└── mobile/ optional
├── base_pose (7,)
└── base_twist (6,), [vx, vy, vz, wx, wy, wz]
```
</details>
<details>
<summary>Trajectory Data Format</summary>
```text
Trajectory Data Format
├── data_format_version string, e.g. "v1.0"
├── instruction / instructions string, or JSON-serialized list[str]
├── subtasks JSON-serialized annotations, optional
├── additional_info/
│ └── frequency int
├── vision/
│ ├── cam_head/
│ │ ├── colors (T, H, W, 3), uint8 RGB or encoded stream
│ │ ├── depths (T, H, W) or (T, H, W, 1), optional
│ │ ├── intrinsic_matrix (3, 3) or (T, 3, 3), optional
│ │ ├── extrinsic_matrix (4, 4) or (T, 4, 4), optional
│ │ └── shape (2,) or (3,), optional
│ ├── cam_left_wrist/ optional
│ ├── cam_right_wrist/ optional
│ ├── cam_wrist/ optional for single-arm robots
│ └── cam_third_view/ optional
├── action/ action targets, same key naming as state/ below
└── state/
├── left_arm_joint_states (T, DOF), optional
├── left_ee_joint_states (T, EEF_DOF), optional
├── left_ee_poses (T, 7), optional
├── left_tcp_poses (T, 7), optional
├── left_delta_ee_poses (T, 7), optional
├── right_arm_joint_states (T, DOF), optional
├── right_ee_joint_states (T, EEF_DOF), optional
├── right_ee_poses (T, 7), optional
├── right_tcp_poses (T, 7), optional
├── right_delta_ee_poses (T, 7), optional
├── arm_joint_states (T, DOF), optional for single-arm robots
├── ee_joint_states (T, EEF_DOF), optional for single-arm robots
├── ee_poses (T, 7), optional for single-arm robots
├── tcp_poses (T, 7), optional for single-arm robots
├── delta_ee_poses (T, 7), optional for single-arm robots
└── mobile/ optional
├── base_poses (T, 7)
└── base_twists (T, 6), [vx, vy, vz, wx, wy, wz]
```
</details>
Useful converter helpers:
```python
from XPolicyLab.utils.load_file import load_hdf5
from XPolicyLab.utils.process_data import decode_image_bit, get_robot_action_dim_info
```
`decode_image_bit` turns encoded image streams into arrays and returns already-decoded values untouched. `get_robot_action_dim_info(env_cfg_type)` returns robot-specific `arm_dim` and `ee_dim` lists, so adapters do not need to hard-code action dimensions.
Offline code — conversion scripts and training dataloaders — must decode through `decode_image_bit` and never through hand-rolled `cv2.imdecode` / `np.frombuffer` / PIL, because RoboTwin and RoboDojo store image bits in legacy layouts that only this function reads correctly. Runtime code does not decode at all; the policy server has already done it, as noted in [Framework Overview](#-framework-overview). Breaking either rule fails silently and is hard to debug.
[CONTRIBUTING.md](CONTRIBUTING.md#modelpy) states both rules in full, along with the two narrow exceptions to the RGB rule and how a new robot gets registered in both `_robot_info.json` files.
## 💾 Data And Checkpoints
Training and data prep usually name things predictably so eval can find them without guesswork:
```text
<bench_name>-<ckpt_name>-<env_cfg_type>-<action_type>
<bench_name>-<ckpt_name>-<env_cfg_type>-<action_type>-<seed>
```
So if you trained with `bench_name=RoboDojo`, `ckpt_name=cotrain`, `env_cfg_type=arx_x5`, `action_type=joint`, `seed=0`, the run lands in `checkpoints/RoboDojo-cotrain-arx_x5-joint-0/`. How `ckpt_name` maps back to these folders at eval time is covered in [Common Workflow](#-common-workflow).
Policies may also use upstream-native layouts or explicit paths in `deploy.yml`. Check the policy README before assuming a naming convention. For a small local dataset to play with, see [Quick Start](#-quick-start).
## 🤝 Add Your Own Policy
Community policies are welcome — open a PR that adds `policy/<POLICY>/`. A PR is also **required** to enter the official [RoboDojo](https://robodojo-benchmark.com/LeaderBoard) and [RoboTwin](https://robotwin-platform.github.io/leaderboard) leaderboards, together with the checkpoint that reproduces your results. [CONTRIBUTING.md](CONTRIBUTING.md) is the full standard: required files, the `Model` contract, `deploy.yml` keys, script conventions, and the PR template.
The fastest route is to copy the reference adapter, keep the XPolicyLab boundary small, and debug before touching a simulator:
1. **Read [policy/demo_policy](policy/demo_policy/README.md)** — `model.py`, `deploy.py`, `deploy.yml`, and the `eval.sh` / `setup_eval_policy_server.sh` / `setup_eval_env_client.sh` trio.
2. **Scaffold** with `bash scripts/create_policy.sh <POLICY_NAME>`, then fill in its README.
3. **Implement `model.py` first**, keeping `bench_name`, `task_name`, `ckpt_name`, `env_cfg_type`, `action_type`, and `seed` consistent across data, training, and eval ([Common Workflow](#-common-workflow)).
4. **Put runtime defaults in `deploy.yml`** and keep `deploy.py` aligned with `demo_policy/deploy.py` unless the environment loop truly differs.
5. **Run the checks below**, then move to `EVAL_ENV_TYPE=sim` or a [split-machine deployment](#-deployment-flow).
Eval-only submissions are accepted when training code cannot be open-sourced yet: say so in the PR, notify the maintainers ([Contact](#-contact)), and share a timeline. For leaderboard evaluation, attach a checkpoint download script (Hugging Face or ModelScope preferred).
### Checks before a PR
Static checks from the repo root, then the adapter wiring check from `policy/<POLICY>/` — no simulator required:
```bash
git diff --check
bash -n policy/<POLICY>/*.sh
python -m py_compile policy/<POLICY>/model.py policy/<POLICY>/deploy.py
```
```bash
cd policy/<POLICY>
export EVAL_ENV_TYPE=debug
bash eval.sh RoboDojo stack_bowls demo arx_x5 joint 0 0 0 \
<policy_env_or_uv_path> <eval_env_conda_env>
```
This verifies imports, server startup, observation serialization, action keys, action dimensions, and batch logic. The debug client sends plain image arrays by default; re-run with `DEBUG_OBS_ENCODED=1` to make it send encoded camera colors instead — a JPEG buffer, raw bytes, and a plain array across the three cameras — which exercises the server-side decode path that real environment clients rely on. For a quick smoke test, `policy/demo_policy` accepts placeholder env names such as `base`.
<details>
<summary>Using a coding agent</summary>
This repo ships two Agent Skills under [.agents/skills](.agents/skills), which `.cursor/skills` and `.claude/skills` symlink to, so Cursor, Claude Code and Codex all pick them up automatically: `xpolicylab-model-integration` builds an adapter (a prompt like "Integrate <POLICY_NAME> into XPolicyLab" is enough), and `xpolicylab-adapter-check` audits one against [CONTRIBUTING.md](CONTRIBUTING.md) before a PR ("Check policy/<POLICY_NAME>"). [AGENTS.md](AGENTS.md) carries the always-on rules every agent must follow. For an agent that supports none of these, paste this checklist:
```text
Integrate <POLICY_NAME> into XPolicyLab.
Use policy/demo_policy as the reference.
1. Inspect the upstream model's inference API and dependencies.
2. Create or update policy/<POLICY_NAME>/README.md with install, checkpoint, train, and eval commands.
3. Implement install.sh and, if needed, process_data.sh and train.sh.
4. Implement model.py with Model.__init__, update_obs, get_action, reset, and batch methods.
5. Keep deploy.py aligned with policy/demo_policy/deploy.py.
6. Put runtime defaults in deploy.yml, keeping the standard key set (protocol: ws, host, port, ...).
7. Run EVAL_ENV_TYPE=debug eval.sh and fix shape/action-key/server errors.
8. Summarize supported action_type, env_cfg_type, checkpoint layout, and remaining limitations.
```
</details>
## 📝 Citation
If XPolicyLab helps your research, please cite:
```bibtex
@article{community2026xpolicylab,
title={{XPolicyLab}: A Unified Standard and Open Ecosystem for Robot Policy Evaluation and Deployment},
author={Community, XPolicyLab and Chen, Tianxing and Chen, Yue and Nian, Tian and Cai, Zijian and Chen, Guangyu and Lin, Wenwei and Liang, Qiwei and Xiang, Peicheng and Su, Kailun and others},
journal={arXiv preprint arXiv:2608.09892},
year={2026}
}
```
## 📬 Contact
Tianxing Chen (project lead): [chentianxing2002@gmail.com](mailto:chentianxing2002@gmail.com)
A collaborative open-source project led by **MMLab@HKU** and **THU**.
**Core Lead Authors**: Tianxing Chen, Yue Chen, Tian Nian, Zijian Cai, Guangyu Chen, Wenwei Lin, Qiwei Liang.
The full contributor list — spanning every integrated policy — lives on the [project website](https://xpolicylab.github.io/).

View File

@ -0,0 +1,7 @@
# XPolicyLab.py
# Make this module behave like a package so `XPolicyLab.<submodule>` works.
import os as _os
# Tell Python to search submodules under this directory
__path__ = [_os.path.dirname(__file__)]

View File

Binary file not shown.

After

Width:  |  Height:  |  Size: 168 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 139 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 110 KiB

View File

@ -0,0 +1,34 @@
"""Env↔policy transport over WebSocket (default) and legacy TCP."""
from __future__ import annotations
from typing import Any
__all__ = [
"ModelClient",
"ModelServer",
"ModelServerConfig",
"PolicyServer",
"PolicyServerConfig",
"WsModelClient",
]
def __getattr__(name: str) -> Any:
if name in ("ModelClient", "WsModelClient"):
from client_server.ws.model_client import WsModelClient
return WsModelClient
if name in ("ModelServer", "PolicyServer"):
from client_server.ws.model_server import PolicyServer
if name == "ModelServer":
return PolicyServer
return PolicyServer
if name in ("ModelServerConfig", "PolicyServerConfig"):
from client_server.ws.model_server import PolicyServerConfig
if name == "ModelServerConfig":
return PolicyServerConfig
return PolicyServerConfig
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

View File

@ -0,0 +1,6 @@
"""TCP env↔policy transport."""
from client_server.tcp.model_client import ModelClient
from client_server.tcp.model_server import ModelServer
__all__ = ["ModelClient", "ModelServer"]

View File

@ -0,0 +1,164 @@
from .utils import *
import socket
import sys
import time
RED = "\033[31m"
GREEN = "\033[32m"
BLUE = "\033[34m"
YELLOW = "\033[33m"
BOLD = "\033[1m"
RESET = "\033[0m"
def _status(level, color, message):
print(f"{color}{BOLD}[{level}]{RESET} {message}", flush=True)
# Cold-start budget: a policy server loading a large checkpoint has not opened
# its port yet, so the client sees connection refused and retries.
# 180 x 5 s = 15 min, matching the websocket client.
CONNECT_MAX_ATTEMPTS = 180
CONNECT_RETRY_DELAY_S = 5
class ModelClient:
def __init__(self, host="localhost", port=9999, timeout=30):
self.host = host
self.port = port
self.timeout = timeout
self.sock = None
self._connect()
def _connect(self):
attempts = 0
max_attempts = CONNECT_MAX_ATTEMPTS
retry_delay = CONNECT_RETRY_DELAY_S
while attempts < max_attempts:
try:
_status(
"CONNECTING",
BLUE,
f"legacy TCP policy server {self.host}:{self.port} "
f"(attempt {attempts + 1}/{max_attempts}, timeout={self.timeout}s)",
)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.settimeout(self.timeout)
self.sock.connect((self.host, self.port))
_status("CONNECTED", GREEN, f"legacy TCP policy server connected: {self.host}:{self.port}")
return
except Exception as e:
attempts += 1
if self.sock:
self.sock.close()
self.sock = None
if attempts < max_attempts:
_status("CONNECT-FAILED", YELLOW, f"attempt {attempts}/{max_attempts} failed: {e}")
if not sys.stdout.isatty():
# Avoid flooding redirected logs with carriage-return updates.
_status("RECONNECT", YELLOW, f"retrying legacy TCP connection in {retry_delay}s")
time.sleep(retry_delay)
else:
for left in range(retry_delay, 0, -1):
print(
f"\r{YELLOW}{BOLD}[RECONNECT]{RESET} retrying legacy TCP connection in {left:2d}s",
end="",
flush=True,
)
time.sleep(1)
print("\r" + " " * 96 + "\r", end="", flush=True)
else:
_status("ERROR", RED, f"failed to connect to legacy TCP policy server {self.host}:{self.port}")
raise ConnectionError(f"Failed to connect to server after {max_attempts} attempts: {str(e)}")
def _send(self, data):
try:
# Serialize with numpy support
json_data = numpy_to_json(data).encode("utf-8")
# Send data length and data
self.sock.sendall(len(json_data).to_bytes(4, "big"))
self.sock.sendall(json_data)
except Exception as e:
self.close()
raise ConnectionError(f"Communication error: {str(e)}")
def _send_recv(self, data):
"""Send request and receive response with numpy array support"""
try:
# Serialize with numpy support
json_data = numpy_to_json(data).encode("utf-8")
# Send data length and data
self.sock.sendall(len(json_data).to_bytes(4, "big"))
self.sock.sendall(json_data)
# Receive and deserialize response
response = self._recv_response()
return response
except Exception as e:
self.close()
raise ConnectionError(f"Communication error: {str(e)}")
def _recv_response(self):
"""Receive response with numpy array reconstruction"""
# Read response length
len_data = self.sock.recv(4)
if not len_data:
raise ConnectionError("Connection closed by server")
size = int.from_bytes(len_data, "big")
# Read complete response
chunks = []
received = 0
while received < size:
chunk = self.sock.recv(min(size - received, 4096))
if not chunk:
raise ConnectionError("Incomplete response received")
chunks.append(chunk)
received += len(chunk)
# Deserialize with numpy reconstruction
return json_to_numpy(b"".join(chunks).decode("utf-8"))
def call(self, func_name=None, obs=None, _reconnect_attempted=False):
request = {"cmd": func_name, "obs": obs}
try:
response = self._send_recv(request)
except ConnectionError:
if _reconnect_attempted:
raise
_status(
"RECONNECT",
YELLOW,
f"legacy TCP connection dropped during {func_name}; reconnecting and retrying once",
)
self._connect()
return self.call(func_name=func_name, obs=obs, _reconnect_attempted=True)
if "res" in response.keys():
return response["res"]
return None
def close(self):
"""Close the connection"""
if self.sock:
try:
self.sock.close()
except:
pass
finally:
self.sock = None
_status("DISCONNECTED", YELLOW, "legacy TCP connection closed")
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
if __name__ == "__main__":
ModelClient()

View File

@ -0,0 +1,122 @@
import socket
import threading
import traceback
from .utils import *
import time
from XPolicyLab.utils.process_data import decode_obs_images
class ModelServer:
def __init__(self, model, host="localhost", port=None):
self.model = model
self.host = host
self.port = port
self.server_socket = None
self.running = False
self.wait_interval = 10
self.client_threads = []
def start(self):
"""Start the model server and listen for incoming client connections"""
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.host, self.port))
self.server_socket.settimeout(self.wait_interval)
self.server_socket.listen(5)
self.running = True
print(f"🚀 Model server started on {self.host}:{self.port}")
print("🔄 Server is waiting for client connections...")
self._accept_connections()
def stop(self):
"""Stop the server and clean up resources gracefully"""
self.running = False
if self.server_socket:
try:
self.server_socket.close()
except:
pass
for t in self.client_threads:
t.join(timeout=1)
print("🛑 Server has been stopped")
def _accept_connections(self):
"""Accept and handle new client connections"""
while self.running:
try:
client_socket, addr = self.server_socket.accept()
print(f"✅ Client connected from {addr}")
# Handle each client in a separate thread
t = threading.Thread(target=self._handle_client, args=(client_socket,), daemon=True)
t.start()
self.client_threads.append(t)
except socket.timeout:
continue
except Exception as e:
if self.running:
print(f"⚠️ Error accepting connection: {e}")
break
def _handle_client(self, client_socket):
"""Process requests from a single client"""
with client_socket:
while self.running:
try:
# Read message length header (4 bytes, big-endian)
len_bytes = client_socket.recv(4)
if not len_bytes:
print("🔌 Client disconnected")
break
msg_length = int.from_bytes(len_bytes, "big")
# Read the full message based on length
chunks = []
remaining = msg_length
while remaining > 0:
chunk = client_socket.recv(min(remaining, 4096))
if not chunk:
raise ConnectionError("Incomplete data received")
chunks.append(chunk)
remaining -= len(chunk)
raw_msg = b"".join(chunks).decode("utf-8")
# Deserialize JSON to Python, reconstruct any numpy arrays
data = json_to_numpy(raw_msg)
# data = pickle.loads(raw_msg)
# Extract command and observation
cmd = data.get("cmd")
obs = data.get("obs") # None if not provided
# Decode every payload-carrying command, matching the ws
# server: custom RPCs can ship camera frames too, and
# decode_obs_images leaves non-observation payloads alone.
if obs is not None:
obs = decode_obs_images(obs)
# Find corresponding model method
method = getattr(self.model, cmd, None)
if not callable(method):
raise AttributeError(f"No model method named '{cmd}'")
# Call method with or without obs
st = time.monotonic()
result = method(obs) if obs is not None else method()
response = {"res": result}
# Serialize response and send back with length header
resp_bytes = numpy_to_json(response).encode("utf-8")
client_socket.sendall(len(resp_bytes).to_bytes(4, "big"))
client_socket.sendall(resp_bytes)
except (ConnectionResetError, BrokenPipeError):
print("🔌 Client connection lost")
break
except Exception as e:
err = f"Error handling request: {e}"
print(f"⚠️ {err}")
tb = traceback.format_exc()
error_resp = numpy_to_json({"error": err, "traceback": tb}).encode("utf-8")
client_socket.sendall(len(error_resp).to_bytes(4, "big"))
client_socket.sendall(error_resp)
break

View File

@ -0,0 +1,75 @@
import json
import numpy as np
import base64
from typing import Any, Mapping
try:
import torch
_HAS_TORCH = True
except Exception:
_HAS_TORCH = False
def _to_numpy(obj: Any) -> Any:
"""递归将 torch.Tensor / numpy 转成可 JSON 序列化类型"""
if _HAS_TORCH and isinstance(obj, torch.Tensor):
# Handle CUDA and half precision by moving to CPU before converting to NumPy
return obj.detach().cpu().numpy()
if isinstance(obj, np.ndarray):
return obj # Let NumpyEncoder handle this
if isinstance(obj, np.generic):
return obj.item()
if isinstance(obj, bytes):
# Encode bytes as base64 so the payload stays JSON-serializable
return {"__bytes__": True, "data": base64.b64encode(obj).decode("ascii")}
if isinstance(obj, Mapping):
return {k: _to_numpy(v) for k, v in obj.items()}
if isinstance(obj, (list, tuple)):
return [_to_numpy(v) for v in obj]
# Try direct serialization first; fall back to str on failure
try:
json.dumps(obj)
return obj
except Exception:
return str(obj)
class NumpyEncoder(json.JSONEncoder):
"""支持 numpy array 序列化"""
def default(self, obj):
if isinstance(obj, np.ndarray):
dtype = str(obj.dtype)
return {
"__numpy_array__": True,
"data": base64.b64encode(obj.tobytes()).decode("ascii"),
"dtype": dtype,
"shape": obj.shape,
}
elif isinstance(obj, (np.integer, np.int32, np.int64)):
return int(obj)
elif isinstance(obj, (np.floating, np.float32, np.float64)):
return float(obj)
elif isinstance(obj, np.bool_):
return bool(obj)
return super().default(obj)
def numpy_to_json(data: Any) -> str:
"""将任意嵌套结构转换为 JSON 字符串"""
cleaned = _to_numpy(data)
return json.dumps(cleaned, cls=NumpyEncoder, ensure_ascii=False)
def json_to_numpy(json_str: str) -> Any:
"""从 JSON 字符串反序列化 numpy array 和 bytes"""
def object_hook(dct):
if "__numpy__" in dct:
data = base64.b64decode(dct["__numpy__"])
return np.frombuffer(data, dtype=np.dtype(dct["dtype"])).reshape(dct["shape"])
if "__numpy_array__" in dct:
data = base64.b64decode(dct["data"])
return np.frombuffer(data, dtype=np.dtype(dct["dtype"])).reshape(dct["shape"])
if "__bytes__" in dct:
return base64.b64decode(dct["data"])
return dct
return json.loads(json_str, object_hook=object_hook)

View File

@ -0,0 +1,40 @@
"""WebSocket env↔policy transport."""
from __future__ import annotations
from typing import Any
from client_server.ws.model_client import WsModelClient
from client_server.ws.protocol.client import PolicyEvalClient, PolicyEvalClientConfig
from client_server.ws.protocol.exceptions import (
ErrorCode,
ServerRestartedError,
WsError,
)
from client_server.ws.protocol.messages import MessageType
from client_server.ws.protocol.schemas import Frame
__all__ = [
"ErrorCode",
"Frame",
"MessageType",
"PolicyEvalClient",
"PolicyEvalClientConfig",
"PolicyServer",
"PolicyServerConfig",
"ServerRestartedError",
"WsError",
"WsModelClient",
]
def __getattr__(name: str) -> Any:
if name == "PolicyServer":
from client_server.ws.model_server import PolicyServer
return PolicyServer
if name == "PolicyServerConfig":
from client_server.ws.model_server import PolicyServerConfig
return PolicyServerConfig
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

View File

@ -0,0 +1,4 @@
from client_server.ws.model_server import main
if __name__ == "__main__":
main()

View File

@ -0,0 +1,219 @@
"""Synchronous environment-side adapter for the policy websocket protocol."""
from __future__ import annotations
import asyncio
import threading
from collections.abc import Coroutine
from typing import Any
from client_server.ws.protocol.client import PolicyEvalClient, PolicyEvalClientConfig
class WsModelClient:
"""Synchronous env-side client for the websocket policy protocol.
The bundled event loop runs on a dedicated background thread so
websocket keepalive pings are answered even while the calling thread
does long blocking work (sim stepping, video encoding) between policy
calls — otherwise the server's ping timeout would drop the connection
whenever the env blocks longer than ping_interval + ping_timeout.
Not thread-safe: issue calls from a single thread at a time (concurrent
calls would interleave requests and the reconnect logic on one client).
The `step` sent on each frame counts INFERENCE calls (get_action /
get_action_batch), not environment control steps — one inference call
yields a whole action chunk. The server currently only echoes it back.
"""
def __init__(
self,
*,
url: str,
evaluation_id: str,
trial_id: str,
action_case_id: str | None = None,
repeat_index: int | None = None,
ws_ping_interval_s: float | None = 20.0,
ws_ping_timeout_s: float | None = 20.0,
connect_timeout_s: float | None = None,
handshake_timeout_s: float | None = None,
request_timeout_s: float | None = None,
max_connect_attempts: int | None = None,
connect_retry_delay_s: float | None = None,
max_connect_seconds: float | None = None,
close_timeout_s: float | None = None,
client: Any | None = None,
):
self.action_case_id = action_case_id
self.trial_id = trial_id
self.repeat_index = repeat_index
self._step = 0
# None means "keep the PolicyEvalClientConfig default", so the defaults
# stay defined in exactly one place.
tuning = {
"connect_timeout_s": connect_timeout_s,
"handshake_timeout_s": handshake_timeout_s,
"request_timeout_s": request_timeout_s,
"max_connect_attempts": max_connect_attempts,
"connect_retry_delay_s": connect_retry_delay_s,
"max_connect_seconds": max_connect_seconds,
"close_timeout_s": close_timeout_s,
}
self._client = client or PolicyEvalClient(
PolicyEvalClientConfig(
url=url,
evaluation_id=evaluation_id,
ws_ping_interval_s=ws_ping_interval_s,
ws_ping_timeout_s=ws_ping_timeout_s,
**{k: v for k, v in tuning.items() if v is not None},
)
)
# Start the loop thread only after config/client construction: a
# validation error above must not leak a running thread + open loop.
self._loop = asyncio.new_event_loop()
self._loop_thread = threading.Thread(
target=self._loop.run_forever, name="ws-model-client", daemon=True
)
self._loop_thread.start()
try:
self._run(self._client.connect(handshake=True))
except BaseException:
# Don't leak the loop thread when the server is unreachable.
self._shutdown_loop()
raise
def _run(self, coro: Coroutine[Any, Any, Any]) -> Any:
"""Run a client coroutine on the background loop and wait for it."""
if self._loop.is_closed() or not self._loop_thread.is_alive():
# Fail loudly instead of hanging forever on a future no loop
# will ever complete (thread died or close() already ran).
coro.close()
raise RuntimeError("websocket client event loop is not running")
return asyncio.run_coroutine_threadsafe(coro, self._loop).result()
def _shutdown_loop(self) -> None:
if self._loop.is_closed():
return
if self._loop_thread.is_alive():
# Cancel and drain leftover tasks (e.g. an orphaned reconnect
# after KeyboardInterrupt) so loop.close() below does not emit
# "Task was destroyed but it is pending" warnings.
async def _drain() -> None:
tasks = [
t
for t in asyncio.all_tasks()
if t is not asyncio.current_task()
]
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
try:
asyncio.run_coroutine_threadsafe(_drain(), self._loop).result(
timeout=5.0
)
except Exception:
pass
self._loop.call_soon_threadsafe(self._loop.stop)
self._loop_thread.join(timeout=5.0)
if not self._loop_thread.is_alive():
self._loop.close()
def call(self, func_name: str | None = None, obs: Any = None, **kwargs: Any) -> Any:
# Payload channel is only `obs` (legacy TCP compatibility). Extra kwargs
# such as env_idx_list=... are not forwarded — reject them loudly.
if kwargs:
unexpected = ", ".join(sorted(kwargs))
raise TypeError(
f"WsModelClient.call() only accepts func_name and obs; "
f"unsupported keyword argument(s): {unexpected}. "
f"For get_action_batch, pass env indices as obs=env_idx_list."
)
if func_name == "prepare_case":
if self.action_case_id is None:
raise ValueError("prepare_case requires action_case_id")
response = self._run(
self._client.prepare_case(
self.action_case_id,
case_meta=self._dict_payload(func_name, obs),
repeat_index=self.repeat_index,
)
)
return response.payload.get("result")
if func_name == "reset":
# model.reset() takes no arguments; reject a payload instead of
# shipping one the server would have to drop.
if obs is not None:
raise TypeError("reset takes no obs payload")
self._step = 0
response = self._run(
self._client.reset(
trial_id=self.trial_id,
action_case_id=self.action_case_id,
repeat_index=self.repeat_index,
)
)
return response.payload.get("result")
if func_name == "trial_end":
response = self._run(
self._client.trial_end(
trial_id=self.trial_id,
action_case_id=self.action_case_id,
result=self._dict_payload(func_name, obs),
repeat_index=self.repeat_index,
)
)
return response.payload.get("result")
if not isinstance(func_name, str) or not func_name or func_name.startswith("_"):
raise NotImplementedError(f"unsupported websocket model call: {func_name!r}")
# Generic model-method dispatch, matching the legacy TCP client which
# forwarded any func_name. This covers update_obs/update_obs_batch
# (obs / obs list), get_action (no obs), get_action_batch
# (obs=env_idx_list), and custom public methods some policies expose
# (e.g. Mem_0's begin_episode/step).
response = self._run(
self._client.call(
func_name,
obs,
trial_id=self.trial_id,
action_case_id=self.action_case_id,
repeat_index=self.repeat_index,
step=self._step,
)
)
if func_name in {"get_action", "get_action_batch"}:
self._step += 1
return response.payload.get("result")
@staticmethod
def _dict_payload(func_name: str, obs: Any) -> dict | None:
# Reject loudly instead of silently dropping a mistyped payload.
if obs is not None and not isinstance(obs, dict):
raise TypeError(
f"{func_name} payload must be a dict or None, "
f"got {type(obs).__name__}"
)
return obs
def close(self) -> None:
if self._loop.is_closed():
return
try:
if self._loop_thread.is_alive():
self._run(self._client.close())
finally:
self._shutdown_loop()
def __enter__(self) -> WsModelClient:
return self
def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
self.close()

View File

@ -0,0 +1,635 @@
"""WebSocket policy server."""
from __future__ import annotations
import argparse
import asyncio
import importlib
import inspect
import logging
import time
from collections import OrderedDict
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Any
from uuid import uuid4
import yaml
from websockets.asyncio.server import Server, ServerConnection, serve
from websockets.exceptions import ConnectionClosed
from client_server.ws.protocol.codec import decode_envelope, decode_frame, encode_frame
from client_server.ws.protocol.exceptions import ErrorCode, WsError
from client_server.ws.protocol.keepalive import normalize_ws_ping
from client_server.ws.protocol.messages import MessageType
from client_server.ws.protocol.schemas import Frame
from XPolicyLab.utils.process_data import decode_obs_images
logger = logging.getLogger(__name__)
# Bound on cached responses kept for duplicate-request replay. The client
# retries a request at most once and immediately, so the replay window is a
# handful of in-flight requests; keep the cap small because each entry holds
# a full response frame (potentially a whole action-chunk result).
_RESPONSE_CACHE_SIZE = 256
# Bound on concurrently processed frames per connection. The bundled client
# is synchronous (one outstanding request), so this only protects against
# misbehaving peers flooding frames faster than the model can answer.
_MAX_INFLIGHT_RESPONSES = 128
# How long shutdown paths wait for still-running model calls before giving
# up. A hung model must not be able to block stop() (or a disconnecting
# client's connection handler) forever.
_SHUTDOWN_DRAIN_TIMEOUT_S = 30.0
def _ok_payload(result: Any = None) -> dict[str, Any]:
payload: dict[str, Any] = {"ok": True}
if result is not None:
payload["result"] = result
return payload
@dataclass
class PolicyServerConfig:
host: str = "0.0.0.0"
port: int = 19000
ws_ping_interval_s: float | None = 20.0
ws_ping_timeout_s: float | None = 20.0
def __post_init__(self) -> None:
self.ws_ping_interval_s = normalize_ws_ping(
self.ws_ping_interval_s, field_name="ws_ping_interval_s"
)
self.ws_ping_timeout_s = normalize_ws_ping(
self.ws_ping_timeout_s, field_name="ws_ping_timeout_s"
)
@dataclass
class PolicyServer:
model: Any
config: PolicyServerConfig = field(default_factory=PolicyServerConfig)
_server: Server | None = field(default=None, init=False)
_model_lock: asyncio.Lock = field(
default_factory=asyncio.Lock,
init=False,
repr=False,
)
# Identifies this server process in HELLO_ACK; the client pins it and
# refuses to continue if a reconnect lands on a different process (a
# restarted server lost the model state and this cache).
_instance_id: str = field(default_factory=lambda: uuid4().hex, init=False)
# request_id -> (request type, response frame), for exactly-once semantics
# across client reconnect retries (the client reuses the request_id when
# retrying). The request type is kept to detect id reuse across types.
_response_cache: OrderedDict[str, tuple[MessageType, Frame]] = field(
default_factory=OrderedDict, init=False, repr=False
)
# request_id -> (request type, running execution). A retry can arrive while
# the original attempt is STILL executing (e.g. long inference on the old
# connection); the duplicate awaits the same task instead of running twice.
_inflight: dict[str, tuple[MessageType, asyncio.Task[Frame | None]]] = field(
default_factory=dict, init=False, repr=False
)
async def start(self) -> None:
if self._server is not None:
return
self._server = await serve(
self._handle_connection,
self.config.host,
self.config.port,
max_size=None,
# An observation frame is multi-MB of camera pixels (or already
# JPEG-compressed buffers). Deflating that on every step costs
# real CPU on the event loop thread for almost no size win, and
# the stall delays the keepalive pongs.
compression=None,
ping_interval=self.config.ws_ping_interval_s,
ping_timeout=self.config.ws_ping_timeout_s,
)
logger.info("websocket policy server listening on %s", self.url)
async def stop(self) -> None:
if self._server is None:
return
self._server.close()
await self._server.wait_closed()
self._server = None
# Executions are shielded from their waiters' cancellation, so closing
# the server can leave them running; drain them instead of letting the
# loop tear down pending tasks. Bounded: a model call that hangs must
# not turn stop() into a hang too.
inflight = [task for _type, task in self._inflight.values()]
if inflight:
_done, still_running = await asyncio.wait(
inflight, timeout=_SHUTDOWN_DRAIN_TIMEOUT_S
)
if still_running:
logger.warning(
"%d model call(s) still running %ss after stop(); abandoning them",
len(still_running),
_SHUTDOWN_DRAIN_TIMEOUT_S,
)
async def serve_forever(self) -> None:
await self.start()
assert self._server is not None
try:
await self._server.serve_forever()
finally:
# Runs on cancellation too (e.g. Ctrl-C under asyncio.run), so
# shielded model calls get their bounded drain instead of being
# torn down mid-execution by loop shutdown.
await self.stop()
@property
def url(self) -> str:
port = self.config.port
if self._server is not None and self._server.sockets:
port = self._server.sockets[0].getsockname()[1]
return f"ws://{self.config.host}:{port}"
async def _handle_connection(self, websocket: ServerConnection) -> None:
send_lock = asyncio.Lock()
pending: set[asyncio.Task[None]] = set()
async def respond(frame: Frame) -> None:
try:
response = await self.process_frame(frame)
if response is None:
await websocket.close()
return
try:
data = encode_frame(response)
except Exception as exc:
# A result the codec cannot serialize (e.g. a torch tensor
# returned by the model) must fail loudly: without a reply
# the client would stall for its full request timeout. The
# unencodable response was already cached, so overwrite it
# too, or retries would replay the same poisoned entry.
logger.exception("policy server response is not serializable")
response = self._error_reply(
frame,
ErrorCode.INTERNAL,
f"response is not msgpack-serializable: {exc}",
)
self._cache_response(
frame.request_id, frame.message_type, response
)
data = encode_frame(response)
async with send_lock:
await websocket.send(data)
except asyncio.CancelledError:
raise
except ConnectionClosed as exc:
# Expected when the original attempt's connection died
# mid-request; the response is cached for the client's retry.
logger.warning("response undeliverable, client gone: %s", exc)
except Exception:
logger.exception("failed to send policy server response")
try:
async for raw in websocket:
if not isinstance(raw, (bytes, bytearray)):
continue
try:
frame = decode_envelope(bytes(raw))
except WsError as exc:
logger.error("invalid request frame: %s", exc)
# Best-effort ERROR reply: without one the client stalls
# for its full request timeout instead of failing fast.
await self._reject_invalid_frame(
websocket, send_lock, bytes(raw), exc
)
continue
if len(pending) >= _MAX_INFLIGHT_RESPONSES:
# Backpressure: process inline instead of queuing yet
# another task for an overwhelmed connection.
await respond(frame)
continue
task = asyncio.create_task(respond(frame))
pending.add(task)
task.add_done_callback(pending.discard)
except ConnectionClosed:
# Routine for dropped/restarted clients; not a handler failure.
logger.debug("policy client disconnected")
finally:
if pending:
# Let responses finish so their results land in the replay
# cache for the client's retry — but bounded, so a hung model
# cannot pin this connection handler forever.
_done, still_running = await asyncio.wait(
pending, timeout=_SHUTDOWN_DRAIN_TIMEOUT_S
)
for task in still_running:
# Cancelling a respond task only detaches the waiter; the
# shielded execution keeps running and its result is still
# cached for a retry. stop() drains (or abandons) it.
task.cancel()
async def _reject_invalid_frame(
self,
websocket: ServerConnection,
send_lock: asyncio.Lock,
raw: bytes,
exc: WsError,
) -> None:
"""Answer an undecodable request with an ERROR frame when possible.
The envelope failed validation, but the raw msgpack map often still
carries a usable message_id (e.g. unknown message_type, missing
evaluation_id). Without a reply the client waits out its full
request timeout; a truly unparseable blob is silently dropped.
"""
try:
wire = decode_frame(raw)
message_id = wire.get("message_id")
if message_id is None:
return
error = Frame(
message_type=MessageType.ERROR,
request_id=str(message_id),
evaluation_id=str(wire.get("evaluation_id") or ""),
payload={
"code": exc.code.value,
"message": exc.message,
"details": exc.details,
},
)
data = encode_frame(error)
except Exception:
return
try:
async with send_lock:
await websocket.send(data)
except Exception:
logger.debug("failed to send invalid-frame error reply", exc_info=True)
def _cache_response(
self, request_id: str, msg_type: MessageType, response: Frame
) -> None:
cache = self._response_cache
cache[request_id] = (msg_type, response)
cache.move_to_end(request_id)
while len(cache) > _RESPONSE_CACHE_SIZE:
cache.popitem(last=False)
def _reused_id_error(self, frame: Frame, original: MessageType) -> Frame:
logger.error(
"request_id %s reused for %s after %s; refusing to answer",
frame.request_id,
frame.message_type.value,
original.value,
)
return self._error_reply(
frame,
ErrorCode.INVALID_FRAME,
f"request_id already used for a {original.value} request; "
f"request ids must be unique per request",
)
async def process_frame(self, frame: Frame) -> Frame | None:
# Exactly-once replay: a client that lost the response to a request
# (connection dropped mid-flight) retries with the SAME request_id;
# answer from the cache instead of executing the call a second time.
cached = self._response_cache.get(frame.request_id)
if cached is not None:
original_type, response = cached
if original_type != frame.message_type:
# Replaying a mismatched response would be worse than failing.
return self._reused_id_error(frame, original_type)
self._response_cache.move_to_end(frame.request_id)
logger.info(
"duplicate request %s (%s); replaying cached response",
frame.request_id,
frame.message_type.value,
)
return response
return await self._execute_once(frame)
async def _execute_once(self, frame: Frame) -> Frame | None:
entry = self._inflight.get(frame.request_id)
if entry is not None:
original_type, task = entry
if original_type != frame.message_type:
return self._reused_id_error(frame, original_type)
logger.info(
"request %s (%s) already in flight; awaiting its result",
frame.request_id,
frame.message_type.value,
)
else:
task = asyncio.create_task(self._execute_frame(frame))
self._inflight[frame.request_id] = (frame.message_type, task)
task.add_done_callback(
lambda _t, rid=frame.request_id: self._inflight.pop(rid, None)
)
# Shield: a cancellation of THIS waiter (e.g. its connection died)
# must not kill the shared execution for the other waiter.
return await asyncio.shield(task)
async def _execute_frame(self, frame: Frame) -> Frame | None:
try:
response = await self._dispatch_frame(frame)
except WsError as exc:
# The client only receives str(exc), so a model failure wrapped in
# CALL_FAILED/RESET_FAILED/INFER_FAILED would otherwise leave no
# traceback anywhere. Protocol rejections carry no cause and stay
# a one-liner.
logger.error(
"policy server %s request failed: %s",
frame.message_type.value,
exc.message,
exc_info=exc.__cause__ is not None,
)
response = self._error_reply(frame, exc.code, exc.message, exc.details)
except Exception as exc:
logger.exception("policy server request failed")
response = self._error_reply(frame, ErrorCode.INTERNAL, str(exc))
if response is not None:
self._cache_response(frame.request_id, frame.message_type, response)
return response
def _reply(
self,
frame: Frame,
message_type: MessageType,
payload: dict[str, Any] | None = None,
) -> Frame:
return Frame(
message_type=message_type,
request_id=frame.request_id,
evaluation_id=frame.evaluation_id,
action_case_id=frame.action_case_id,
trial_id=frame.trial_id,
repeat_index=frame.repeat_index,
step=frame.step,
payload=payload or {},
)
def _error_reply(
self,
frame: Frame,
code: ErrorCode,
message: str,
details: dict[str, Any] | None = None,
) -> Frame:
return self._reply(
frame,
MessageType.ERROR,
{
"code": code.value,
"message": message,
"details": details or {},
},
)
async def _invoke_method(self, method, *args: Any) -> Any:
if inspect.iscoroutinefunction(method):
return await method(*args)
result = await asyncio.to_thread(method, *args)
if inspect.isawaitable(result):
return await result
return result
async def _call_model_method(self, method, *args: Any) -> Any:
async with self._model_lock:
return await self._invoke_method(method, *args)
async def _dispatch_frame(self, frame: Frame) -> Frame | None:
if frame.message_type == MessageType.HELLO:
return self._reply(
frame,
MessageType.HELLO_ACK,
{
"ok": True,
"server": "xpolicylab_policy_server",
"server_instance_id": self._instance_id,
},
)
if frame.message_type == MessageType.PREPARE_CASE:
return await self._handle_prepare_case(frame)
if frame.message_type == MessageType.RESET:
return await self._handle_reset(frame)
if frame.message_type == MessageType.CALL:
return await self._handle_call(frame)
if frame.message_type == MessageType.INFER:
return await self._handle_infer(frame)
if frame.message_type == MessageType.TRIAL_END:
return await self._handle_trial_end(frame)
if frame.message_type == MessageType.HEARTBEAT:
return self._reply(frame, MessageType.HEARTBEAT_ACK, {"ok": True})
if frame.message_type == MessageType.CLOSE:
return None
raise WsError(
ErrorCode.UNKNOWN_MESSAGE_TYPE,
f"unsupported request type: {frame.message_type.value}",
)
async def _handle_prepare_case(self, frame: Frame) -> Frame:
case_meta = dict(frame.payload)
if frame.action_case_id is not None:
case_meta.setdefault("action_case_id", frame.action_case_id)
hook = getattr(self.model, "prepare_case", None)
hook_result = (
await self._call_model_method(hook, case_meta) if callable(hook) else None
)
return self._reply(
frame, MessageType.PREPARE_CASE_ACK, _ok_payload(hook_result)
)
async def _handle_reset(self, frame: Frame) -> Frame:
method = getattr(self.model, "reset", None)
if not callable(method):
raise WsError(ErrorCode.RESET_FAILED, "model.reset is not callable")
# reset() takes no arguments (see ModelTemplate.reset). The legacy TCP
# server appeared to support reset(obs) only because it dispatched every
# command through one generic `method(obs) if obs is not None` line; no
# caller ever sent one. A policy that needs a first observation should
# reset() and then take a normal update_obs.
try:
result = await self._call_model_method(method)
except Exception as exc:
raise WsError(ErrorCode.RESET_FAILED, str(exc)) from exc
return self._reply(frame, MessageType.RESET_RESULT, _ok_payload(result))
async def _handle_call(self, frame: Frame) -> Frame:
# Generic model-method dispatch, mirroring the legacy TCP server:
# call method(obs) when obs is provided, method() otherwise.
func_name = frame.payload.get("func_name")
if not isinstance(func_name, str) or not func_name or func_name.startswith("_"):
raise WsError(
ErrorCode.INVALID_FRAME,
f"call payload has invalid func_name: {func_name!r}",
)
method = getattr(self.model, func_name, None)
if not callable(method):
raise WsError(
ErrorCode.INVALID_FRAME, f"no model method named {func_name!r}"
)
obs = frame.payload.get("obs")
if func_name in {"update_obs", "update_obs_batch", "infer"} and obs is None:
raise WsError(ErrorCode.INVALID_FRAME, f"{func_name} payload missing obs")
if obs is not None:
try:
# Decode for EVERY payload-carrying call, not just the standard
# method names: adapters are forbidden from decoding themselves,
# and a policy may expose custom RPCs that ship camera frames
# (Mem_0's begin_episode/step). decode_obs_images only touches
# dicts with a `vision` field and returns already-decoded values
# untouched, so payloads that are not observations — notably
# get_action_batch's env_idx_list — pass straight through.
#
# JPEG decode is CPU-bound (a batched obs can be N envs x M
# cameras); run it off the event loop for the same reason
# compression is disabled — a blocked loop delays keepalive
# pongs and every other connection.
obs = await asyncio.to_thread(decode_obs_images, obs)
except ValueError as exc:
raise WsError(ErrorCode.INVALID_FRAME, str(exc)) from exc
start = time.perf_counter()
try:
if obs is None:
result = await self._call_model_method(method)
else:
result = await self._call_model_method(method, obs)
except Exception as exc:
raise WsError(ErrorCode.CALL_FAILED, str(exc)) from exc
payload = _ok_payload(result)
payload["latency_ms"] = (time.perf_counter() - start) * 1000.0
return self._reply(frame, MessageType.CALL_RESULT, payload)
async def _handle_infer(self, frame: Frame) -> Frame:
observation = frame.payload.get("observation")
if observation is None:
raise WsError(ErrorCode.INVALID_FRAME, "infer payload missing observation")
try:
# Off the event loop: see _handle_call — JPEG decode is CPU-bound.
observation = await asyncio.to_thread(decode_obs_images, observation)
except ValueError as exc:
raise WsError(ErrorCode.INVALID_FRAME, str(exc)) from exc
start = time.perf_counter()
try:
async with self._model_lock:
update_obs = getattr(self.model, "update_obs", None)
get_action = getattr(self.model, "get_action", None)
if callable(update_obs) and callable(get_action):
if not inspect.iscoroutinefunction(
update_obs
) and not inspect.iscoroutinefunction(get_action):
# Keep the sync pair on ONE worker thread: some models
# carry thread-affine state between the two calls.
# NOTE: this only holds WITHIN one INFER request;
# across requests the default executor may pick a
# different thread, so threading.local model state is
# not supported.
def run_legacy_infer() -> Any:
update_result = update_obs(observation)
if inspect.isawaitable(update_result):
raise TypeError(
"update_obs returned an awaitable but is not async"
)
result = get_action()
if inspect.isawaitable(result):
raise TypeError(
"get_action returned an awaitable but is not async"
)
return result
result = await asyncio.to_thread(run_legacy_infer)
else:
# Mixed/async pair: _invoke_method keeps sync halves
# off the event loop (running a blocking sync call
# inline here would stall every connection).
await self._invoke_method(update_obs, observation)
result = await self._invoke_method(get_action)
else:
infer = getattr(self.model, "infer", None)
if not callable(infer):
raise AttributeError(
"model must implement update_obs()/get_action() or infer(observation)"
)
result = await self._invoke_method(infer, observation)
except Exception as exc:
raise WsError(ErrorCode.INFER_FAILED, str(exc)) from exc
latency_ms = (time.perf_counter() - start) * 1000.0
if isinstance(result, Mapping) and "actions" in result:
payload: dict[str, Any] = dict(result)
payload.setdefault("latency_ms", latency_ms)
else:
payload = {"actions": result, "latency_ms": latency_ms}
return self._reply(frame, MessageType.INFER_RESULT, payload)
async def _handle_trial_end(self, frame: Frame) -> Frame:
hook = getattr(self.model, "on_trial_end", None)
hook_result = (
await self._call_model_method(hook, dict(frame.payload))
if callable(hook)
else None
)
return self._reply(frame, MessageType.TRIAL_END_ACK, _ok_payload(hook_result))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"--config-path", "--config_path", dest="config_path", required=True
)
parser.add_argument("--protocol", choices=("ws",), default="ws")
parser.add_argument("--host")
parser.add_argument("--port", type=int)
parser.add_argument("--relay-url", dest="relay_url")
args = parser.parse_args()
with open(args.config_path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
if not isinstance(data, dict):
raise ValueError(f"config must be a YAML mapping: {args.config_path}")
deploy_cfg = data
if args.host is not None:
deploy_cfg["host"] = args.host
if args.port is not None:
deploy_cfg["port"] = args.port
if args.relay_url is not None:
deploy_cfg["relay_url"] = args.relay_url
policy_name = deploy_cfg.get("policy_name")
if not policy_name:
raise ValueError("policy_name must be specified in config")
module = importlib.import_module(f"XPolicyLab.policy.{policy_name}.model")
model_class = getattr(module, "Model")
model = model_class(deploy_cfg)
# deploy.yml may carry an explicit `port: null`; treat it as "unset"
# instead of crashing in int(None).
port = deploy_cfg.get("port")
server = PolicyServer(
model,
PolicyServerConfig(
host=deploy_cfg.get("host") or "0.0.0.0",
port=int(port) if port is not None else 19000,
ws_ping_interval_s=deploy_cfg.get("ws_ping_interval_s", 20.0),
ws_ping_timeout_s=deploy_cfg.get("ws_ping_timeout_s", 20.0),
),
)
try:
asyncio.run(server.serve_forever())
except KeyboardInterrupt:
# serve_forever's finally already drained in-flight model calls.
logger.info("policy server interrupted; shut down cleanly")
if __name__ == "__main__":
main()

View File

@ -0,0 +1,23 @@
from client_server.ws.protocol.client import PolicyEvalClient, PolicyEvalClientConfig
from client_server.ws.protocol.codec import decode_envelope, decode_frame, encode_frame
from client_server.ws.protocol.exceptions import (
ErrorCode,
ServerRestartedError,
WsError,
)
from client_server.ws.protocol.messages import REQUEST_RESPONSE_PAIRS, MessageType
from client_server.ws.protocol.schemas import Frame
__all__ = [
"ErrorCode",
"Frame",
"MessageType",
"PolicyEvalClient",
"PolicyEvalClientConfig",
"REQUEST_RESPONSE_PAIRS",
"ServerRestartedError",
"WsError",
"decode_envelope",
"decode_frame",
"encode_frame",
]

View File

@ -0,0 +1,669 @@
"""Eval Client over WebSocket (msgpack frames)."""
from __future__ import annotations
import asyncio
import logging
import math
import sys
from collections.abc import AsyncIterator, Mapping
from dataclasses import dataclass, field
from typing import Any, Literal, Protocol
from uuid import uuid4
import websockets
from client_server.ws.protocol.codec import decode_envelope, encode_frame
from client_server.ws.protocol.exceptions import (
ErrorCode,
ServerRestartedError,
WsError,
)
from client_server.ws.protocol.keepalive import normalize_ws_ping
from client_server.ws.protocol.messages import REQUEST_RESPONSE_PAIRS, MessageType
from client_server.ws.protocol.schemas import Frame
logger = logging.getLogger(__name__)
_RED = "\033[31m"
_GREEN = "\033[32m"
_BLUE = "\033[34m"
_YELLOW = "\033[33m"
_BOLD = "\033[1m"
_RESET = "\033[0m"
def _status(level: str, color: str, message: str) -> None:
print(f"{color}{_BOLD}[{level}]{_RESET} {message}", flush=True)
def _body_with(
field: str, value: str, extra: Mapping[str, Any] | None
) -> dict[str, Any]:
"""Merge a caller payload with one protocol field it must not shadow.
Spreading the payload over the field would let a stray `trial_id` in a
trial result silently retarget the request; spreading it under would
silently discard the caller's value. Refuse the ambiguity instead.
"""
body = dict(extra or {})
if body.get(field, value) != value:
raise ValueError(
f"payload key {field!r}={body[field]!r} conflicts with the "
f"request's {field}={value!r}"
)
body[field] = value
return body
class _ClosedDuringConnect(ConnectionError):
"""close() completed while a connect attempt was mid-flight.
The attempt may have opened a connection close() never saw; the connect
loop must tear it down and stop retrying instead of reviving the client.
"""
def _compact_exception(exc: BaseException, max_len: int = 180) -> str:
text = str(exc).replace("\n", " ").strip()
if not text:
text = exc.__class__.__name__
else:
text = f"{exc.__class__.__name__}: {text}"
if len(text) > max_len:
text = text[: max_len - 3] + "..."
return text
class WebSocketConnection(Protocol):
async def send(self, message: Any, text: bool | None = None) -> None: ...
async def close(self) -> None: ...
def __aiter__(self) -> AsyncIterator[bytes | str]: ...
@dataclass
class PolicyEvalClientConfig:
url: str
evaluation_id: str
connect_timeout_s: float = 30.0
request_timeout_s: float = 120.0
# Handshake budget, separate from request_timeout_s: the server answers
# HELLO without touching the model (it is constructed before the port
# opens), so this is a plain round-trip and must not inherit the timeout
# sized for slow inference — otherwise a hung server stalls every connect
# attempt for minutes. Raise it for servers that do work on HELLO.
handshake_timeout_s: float = 60.0
# Cold-start budget: a policy server loading a large checkpoint has not
# opened its port yet, so the client sees connection refused and retries.
# 180 x 5 s = 15 min, matching the legacy TCP client so both protocols
# tolerate the same slow model load.
max_connect_attempts: int = 180
connect_retry_delay_s: float = 5.0
# Wall-clock cap on the whole retry loop. The attempt count alone assumes
# each attempt fails fast (connection refused). A server that accepts the
# socket but hangs makes every attempt cost connect_timeout_s +
# handshake_timeout_s, which would stretch 180 attempts into hours.
max_connect_seconds: float = 900.0
# Cap on the websocket closing handshake plus TCP teardown. websockets'
# asyncio implementation waits indefinitely on its own.
close_timeout_s: float = 10.0
ws_ping_interval_s: float | None = 20.0
ws_ping_timeout_s: float | None = 20.0
proxy: str | Literal[True] | None = None
def __post_init__(self) -> None:
self.ws_ping_interval_s = normalize_ws_ping(
self.ws_ping_interval_s, field_name="ws_ping_interval_s"
)
self.ws_ping_timeout_s = normalize_ws_ping(
self.ws_ping_timeout_s, field_name="ws_ping_timeout_s"
)
# These all feed timeouts / sleep loops; nan or a negative value
# would silently break the retry math (max_connect_seconds=0 is the
# documented "no wall-clock cap", so 0 stays allowed everywhere).
for name in (
"connect_timeout_s",
"request_timeout_s",
"handshake_timeout_s",
"connect_retry_delay_s",
"max_connect_seconds",
"close_timeout_s",
):
value = getattr(self, name)
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
or value < 0
):
raise ValueError(
f"{name} must be a non-negative finite number, got {value!r}"
)
if self.max_connect_attempts < 1:
raise ValueError(
f"max_connect_attempts must be >= 1, got {self.max_connect_attempts!r}"
)
@dataclass
class PolicyEvalClient:
config: PolicyEvalClientConfig
_ws: WebSocketConnection | None = field(default=None, init=False)
_recv_task: asyncio.Task[None] | None = field(default=None, init=False)
_pending: dict[str, asyncio.Future[Frame]] = field(default_factory=dict, init=False)
_closed: bool = field(default=False, init=False)
_evaluation_plan: dict[str, Any] | None = field(default=None, init=False)
_server_instance_id: str | None = field(default=None, init=False)
async def _sleep_with_countdown(self, seconds: float, label: str) -> None:
# Keep fractional delays intact: truncating 0.5 s to 0 would turn the
# connect-retry loop into a hot loop hammering the server.
if seconds <= 0:
return
interactive = sys.stdout.isatty()
if not interactive:
# Avoid flooding redirected logs with carriage-return updates.
_status("RECONNECT", _YELLOW, f"{label}; retry in {seconds:g}s")
# Sleep in <=1 s slices so close() interrupts the wait promptly
# instead of the retry loop napping through a full delay.
remaining = seconds
while remaining > 0 and not self._closed:
if interactive:
print(
f"\r{_YELLOW}{_BOLD}[RECONNECT]{_RESET} {label}; retry in {remaining:4.1f}s",
end="",
flush=True,
)
step = min(1.0, remaining)
await asyncio.sleep(step)
remaining -= step
if interactive:
print("\r" + " " * 96 + "\r", end="", flush=True)
async def _close_ws(self, ws: WebSocketConnection) -> None:
"""Close one connection without letting an unresponsive peer hang us.
The websockets asyncio implementation waits for the peer to complete
the closing handshake and for TCP teardown, with no close timeout of
its own. Keepalive normally aborts a half-open peer, but that safety
net is gone when ws_ping_interval_s is set to null.
"""
try:
await asyncio.wait_for(ws.close(), timeout=self.config.close_timeout_s)
except asyncio.TimeoutError:
logger.warning(
"websocket close timed out after %ss; abandoning the connection",
self.config.close_timeout_s,
)
except Exception:
pass
async def _reset_connection_state(self) -> None:
# Detach first: the cancelled recv task's cleanup checks identity
# against self._ws, so detaching guarantees it cannot clobber a
# connection installed by a concurrent reconnect.
ws, recv_task = self._ws, self._recv_task
self._ws = None
self._recv_task = None
if ws is not None:
# The recv task is cancelled below, so it can no longer fail the
# futures still waiting on this connection; do it here instead.
self._fail_pending(WsError(ErrorCode.INTERNAL, "connection reset"))
if recv_task is not None:
recv_task.cancel()
if ws is not None:
await self._close_ws(ws)
if recv_task is not None:
# Wait for the recv task to actually finish; otherwise its
# finally block may run after a new connection is installed.
try:
await recv_task
except BaseException:
pass
def _raise_if_closed_during_connect(self) -> None:
# Re-check after every await inside a connect attempt: close() may
# have completed in between, and it cannot see (or close) a
# connection this attempt is still in the middle of establishing.
if self._closed:
raise _ClosedDuringConnect(
"client closed while a connect attempt was in flight"
)
async def connect(
self,
*,
handshake: bool = True,
evaluation_plan: Mapping[str, Any] | dict[str, Any] | None = None,
) -> Frame | None:
if self._ws is not None:
return None
# close() is terminal: do NOT clear _closed here. Clearing it would
# let a connect racing with close() (or called after it) silently
# revive the client and leak a connection close() never saw; the
# loop below refuses instead.
connect_kwargs: dict[str, Any] = {
"max_size": None,
# Must match the server: deflating multi-MB observation frames
# burns CPU on the loop thread for almost no size win.
"compression": None,
"ping_interval": self.config.ws_ping_interval_s,
"ping_timeout": self.config.ws_ping_timeout_s,
}
# `proxy` is only supported by the websockets>=14 asyncio client; passing it
# to older releases (e.g. the legacy client shipped with Isaac Sim's
# websockets 12) raises TypeError inside loop.create_connection.
if self.config.proxy is not None:
connect_kwargs["proxy"] = self.config.proxy
last_err: Exception | None = None
deadline = (
asyncio.get_running_loop().time() + self.config.max_connect_seconds
if self.config.max_connect_seconds > 0
else None
)
attempt = 0
while attempt < self.config.max_connect_attempts:
attempt += 1
if self._closed:
# close() ran while we were retrying (e.g. an orphaned
# background reconnect after the caller gave up); stop
# hammering the server instead of retrying for minutes.
raise ConnectionError("client closed; aborting connect retries")
_status(
"CONNECTING",
_BLUE,
f"websocket policy server {self.config.url} "
f"(attempt {attempt}/{self.config.max_connect_attempts}, timeout={self.config.connect_timeout_s:g}s)",
)
try:
self._ws = await asyncio.wait_for(
websockets.connect(
self.config.url,
**connect_kwargs,
),
timeout=self.config.connect_timeout_s,
)
self._recv_task = asyncio.create_task(self._recv_loop(self._ws))
self._raise_if_closed_during_connect()
logger.info("websocket connected: %s", self.config.url)
_status("CONNECTED", _GREEN, f"websocket policy server connected: {self.config.url}")
if handshake:
ack = await self.hello(evaluation_plan=evaluation_plan)
self._check_server_identity(ack)
self._raise_if_closed_during_connect()
return ack
return None
except (ServerRestartedError, _ClosedDuringConnect):
# Not transient connect failures: either the server came back
# as a different process and lost the model state, or close()
# finished while this attempt was in flight. Tear down whatever
# this attempt opened and do not retry.
await self._reset_connection_state()
raise
except Exception as exc:
last_err = exc
await self._reset_connection_state()
out_of_time = (
deadline is not None
and asyncio.get_running_loop().time() >= deadline
)
if attempt >= self.config.max_connect_attempts or out_of_time:
logger.warning(
"final connect attempt %s/%s failed: %s",
attempt,
self.config.max_connect_attempts,
_compact_exception(exc),
)
break
# During policy cold-start the port may not be ready yet. Keep
# this quiet unless debug logging is enabled; the final failure
# below is the actionable communication error.
logger.debug("connect attempt %s failed", attempt, exc_info=True)
_status(
"WAITING",
_YELLOW,
"policy server not ready "
f"(attempt {attempt}/{self.config.max_connect_attempts}): {_compact_exception(exc)}",
)
await self._sleep_with_countdown(
self.config.connect_retry_delay_s,
f"reconnecting to {self.config.url}",
)
give_up = f"after {attempt} attempts"
if deadline is not None and asyncio.get_running_loop().time() >= deadline:
give_up += f" / {self.config.max_connect_seconds:g}s"
_status("ERROR", _RED, f"failed to connect to {self.config.url} {give_up}")
raise ConnectionError(
f"failed to connect {give_up}: {last_err}"
) from last_err
def _check_server_identity(self, ack: Frame) -> None:
"""Pin the server process identity across reconnects.
The dedupe cache and the model state both live in the server process,
so a reconnect is only safe against the SAME process. A changed (or
freshly appearing) instance id means the server restarted.
"""
server_id = ack.payload.get("server_instance_id")
if not isinstance(server_id, str) or not server_id:
# Pre-instance-id server: cannot detect restarts, keep going.
return
if self._server_instance_id is None:
self._server_instance_id = server_id
elif self._server_instance_id != server_id:
raise ServerRestartedError(
"policy server restarted mid-evaluation; the fresh server "
"lost all model state, so continuing would corrupt results"
)
async def close(self) -> None:
if self._closed:
return
self._closed = True
if self._ws is not None:
try:
# Best-effort protocol-level close so the server sees a clean
# disconnect instead of an aborted TCP connection.
await asyncio.wait_for(
self.send_close(reason="client closing"), timeout=2.0
)
except Exception:
pass
self._fail_pending(WsError(ErrorCode.INTERNAL, "client closed"))
await self._reset_connection_state()
def _fail_pending(self, exc: BaseException) -> None:
for fut in self._pending.values():
if not fut.done():
fut.set_exception(exc)
self._pending.clear()
async def _recv_loop(self, ws: WebSocketConnection) -> None:
close_exc: BaseException | None = None
try:
async for raw in ws:
if not isinstance(raw, (bytes, bytearray)):
continue
try:
frame = decode_envelope(bytes(raw))
except WsError as exc:
logger.error("invalid frame: %s", exc)
continue
self._dispatch_incoming(frame)
except asyncio.CancelledError:
return
except Exception as exc:
logger.error("recv loop ended: %s", exc)
close_exc = exc
finally:
# Only clean up when this loop's connection is still the current
# one. After _reset_connection_state() (or a reconnect) self._ws
# is None or a NEW connection; touching state then would fail
# futures that belong to the new connection.
if self._ws is ws and not self._closed:
self._ws = None
self._fail_pending(
close_exc or WsError(ErrorCode.INTERNAL, "connection closed")
)
# Reaching here means nobody else detached this connection, so
# nobody else will close it either. Safe to await: the cancel
# path returns above with self._ws already detached.
await self._close_ws(ws)
def _dispatch_incoming(self, frame: Frame) -> None:
if frame.message_type == MessageType.ERROR:
err = frame.payload
try:
code = ErrorCode(err.get("code", ErrorCode.INTERNAL.value))
except ValueError:
code = ErrorCode.INTERNAL
exc = WsError(
code,
str(err.get("message", "policy server error")),
details=err.get("details"),
)
req_id = frame.request_id
fut = self._pending.pop(req_id, None)
# done() guard: wait_for() cancels the future on timeout before
# request() pops it from _pending; set_exception on a cancelled
# future raises InvalidStateError and would kill the recv loop.
if fut is not None and not fut.done():
fut.set_exception(exc)
return
req_id = frame.request_id
fut = self._pending.pop(req_id, None)
if fut is not None and not fut.done():
fut.set_result(frame)
async def request(
self,
msg_type: MessageType,
payload: dict[str, Any],
*,
action_case_id: str | None = None,
trial_id: str | None = None,
repeat_index: int | None = None,
step: int = 0,
timeout_s: float | None = None,
_reconnect_attempted: bool = False,
_request_id: str | None = None,
) -> Frame:
expected = REQUEST_RESPONSE_PAIRS.get(msg_type)
if expected is None and msg_type != MessageType.CLOSE:
raise ValueError(f"no response pairing for {msg_type}")
if self._ws is None:
if self._closed:
raise RuntimeError("not connected; client is closed")
if msg_type == MessageType.HELLO:
raise RuntimeError("not connected; call connect() first")
if msg_type == MessageType.CLOSE:
# CLOSE is a courtesy notification; burning the full connect
# retry budget just to say goodbye would be absurd. No
# connection means there is nothing to close.
raise RuntimeError("not connected; nothing to close")
_status(
"RECONNECT",
_YELLOW,
f"connection to {self.config.url} is closed; reconnecting before {msg_type.value}",
)
await self.connect(handshake=True, evaluation_plan=self._evaluation_plan)
# Retries of the same logical request MUST reuse the request_id: the
# server keeps a response cache keyed by it, so a retried request that
# already executed is answered from the cache instead of running twice
# (update_obs/get_action are not idempotent for stateful policies).
request_id = _request_id or str(uuid4())
frame = Frame(
message_type=msg_type,
request_id=request_id,
evaluation_id=self.config.evaluation_id,
action_case_id=action_case_id,
trial_id=trial_id,
repeat_index=repeat_index,
step=step,
payload=payload,
)
# Encode before touching the connection: a payload the codec rejects
# is a caller error and must surface directly, not masquerade as a
# connection failure and trigger a pointless reconnect + retry.
data = encode_frame(frame)
if expected is None:
await self._ws.send(data)
return frame
loop = asyncio.get_running_loop()
fut: asyncio.Future[Frame] = loop.create_future()
self._pending[request_id] = fut
try:
await self._ws.send(data)
except Exception:
self._pending.pop(request_id, None)
if not _reconnect_attempted and not self._closed and msg_type != MessageType.HELLO:
_status("RECONNECT", _YELLOW, f"send failed for {msg_type.value}; reconnecting to {self.config.url}")
await self._reset_connection_state()
await self.connect(handshake=True, evaluation_plan=self._evaluation_plan)
return await self.request(
msg_type,
payload,
action_case_id=action_case_id,
trial_id=trial_id,
repeat_index=repeat_index,
step=step,
timeout_s=timeout_s,
_reconnect_attempted=True,
_request_id=request_id,
)
raise
timeout = timeout_s if timeout_s is not None else self.config.request_timeout_s
try:
response = await asyncio.wait_for(fut, timeout=timeout)
except asyncio.TimeoutError as exc:
self._pending.pop(request_id, None)
# The server may STILL be executing this request. Dedup only
# covers retries with the SAME request_id, so callers must treat
# a timeout as fatal for the trial — re-issuing the logical call
# under a fresh request_id would execute it a second time
# (update_obs/get_action are not idempotent for stateful models).
raise WsError(ErrorCode.TIMEOUT, f"timeout waiting for {expected}") from exc
except asyncio.CancelledError:
# Outer cancellation (e.g. KeyboardInterrupt around
# run_until_complete): wait_for cancels fut, but the _pending
# entry would otherwise leak until the next disconnect.
self._pending.pop(request_id, None)
raise
except Exception:
self._pending.pop(request_id, None)
if self._ws is None and not _reconnect_attempted and not self._closed and msg_type != MessageType.HELLO:
_status(
"RECONNECT",
_YELLOW,
f"connection dropped while waiting for {msg_type.value}; reconnecting to {self.config.url}",
)
await self.connect(handshake=True, evaluation_plan=self._evaluation_plan)
return await self.request(
msg_type,
payload,
action_case_id=action_case_id,
trial_id=trial_id,
repeat_index=repeat_index,
step=step,
timeout_s=timeout_s,
_reconnect_attempted=True,
_request_id=request_id,
)
raise
if expected is not None and response.message_type != expected:
raise WsError(
ErrorCode.INVALID_FRAME,
f"expected {expected.value}, got {response.message_type.value}",
)
return response
async def hello(
self,
*,
evaluation_plan: Mapping[str, Any] | dict[str, Any] | None = None,
) -> Frame:
payload: dict[str, Any] = {}
if evaluation_plan is not None:
# Remember the plan so automatic reconnects can replay the handshake.
self._evaluation_plan = dict(evaluation_plan)
payload["evaluation_plan"] = self._evaluation_plan
return await self.request(
MessageType.HELLO, payload, timeout_s=self.config.handshake_timeout_s
)
async def prepare_case(
self,
action_case_id: str,
case_meta: dict[str, Any] | None = None,
*,
repeat_index: int | None = None,
) -> Frame:
body = _body_with("action_case_id", action_case_id, case_meta)
return await self.request(
MessageType.PREPARE_CASE,
body,
action_case_id=action_case_id,
repeat_index=repeat_index,
)
async def reset(
self,
*,
trial_id: str,
action_case_id: str | None = None,
repeat_index: int | None = None,
) -> Frame:
# No caller payload: model.reset() takes no arguments, so anything
# sent here would only be dropped by the server.
return await self.request(
MessageType.RESET,
{"trial_id": trial_id},
action_case_id=action_case_id,
trial_id=trial_id,
repeat_index=repeat_index,
)
async def call(
self,
func_name: str,
obs: Any = None,
*,
trial_id: str | None = None,
action_case_id: str | None = None,
repeat_index: int | None = None,
step: int = 0,
) -> Frame:
payload: dict[str, Any] = {"func_name": func_name}
if obs is not None:
payload["obs"] = obs
return await self.request(
MessageType.CALL,
payload,
trial_id=trial_id,
action_case_id=action_case_id,
repeat_index=repeat_index,
step=step,
)
async def infer(
self,
observation: dict[str, Any],
*,
trial_id: str | None = None,
action_case_id: str | None = None,
repeat_index: int | None = None,
step: int = 0,
) -> Frame:
return await self.request(
MessageType.INFER,
{"observation": observation},
trial_id=trial_id,
action_case_id=action_case_id,
repeat_index=repeat_index,
step=step,
)
async def trial_end(
self,
*,
trial_id: str,
result: dict[str, Any] | None = None,
action_case_id: str | None = None,
repeat_index: int | None = None,
) -> Frame:
body = _body_with("trial_id", trial_id, result)
return await self.request(
MessageType.TRIAL_END,
body,
trial_id=trial_id,
action_case_id=action_case_id,
repeat_index=repeat_index,
)
async def send_close(self, reason: str = "") -> Frame:
return await self.request(MessageType.CLOSE, {"reason": reason})

View File

@ -0,0 +1,125 @@
"""msgpack encode/decode for WebSocket binary frames."""
from __future__ import annotations
import sys
from typing import Any
import msgpack
import msgpack_numpy
import numpy as np
from pydantic import ValidationError
from client_server.ws.protocol.exceptions import ErrorCode, WsError
from client_server.ws.protocol.schemas import Frame
def _tensor_to_numpy(obj: Any) -> Any:
"""Convert a torch.Tensor to numpy, or return obj unchanged.
torch is looked up in sys.modules rather than imported: only a process
that already loaded torch can hold a tensor, so this keeps the env-side
client from paying a multi-second torch import (and a CUDA context) just
to serialize observations.
"""
torch = sys.modules.get("torch")
if torch is None or not isinstance(obj, torch.Tensor):
return obj
# Same convenience as the legacy TCP codec: models may return CUDA or
# half-precision tensors straight from inference.
tensor = obj.detach().cpu()
if tensor.dtype == torch.bfloat16:
# numpy has no bfloat16; widen instead of failing.
tensor = tensor.float()
return tensor.numpy()
def _encode_numpy(obj: Any) -> Any:
obj = _tensor_to_numpy(obj)
if isinstance(obj, np.ndarray):
if obj.dtype.kind == "O":
raise WsError(
ErrorCode.INVALID_FRAME, "object dtype numpy arrays are not supported"
)
return msgpack_numpy.encode(obj)
if isinstance(obj, np.generic):
if obj.dtype.kind == "O":
raise WsError(
ErrorCode.INVALID_FRAME, "object dtype numpy scalars are not supported"
)
return msgpack_numpy.encode(obj)
def _normalize_map_keys(mapping: dict[Any, Any]) -> dict[Any, Any]:
"""Match the legacy JSON codec: int-keyed maps arrive with str keys.
json.dumps turned ``{0: ...}`` into ``{"0": ...}`` on the wire, so every
model written against the TCP protocol indexes env-idx-keyed maps by
string. msgpack preserves int keys, which would silently change those
lookups from obs["0"] to obs[0]; normalize back to strings instead.
"""
if all(isinstance(key, str) for key in mapping):
return mapping
out: dict[Any, Any] = {}
for key, value in mapping.items():
if isinstance(key, int) and not isinstance(key, bool):
key = str(key)
if key in out:
raise ValueError(
f"duplicate map key after int->str normalization: {key!r}"
)
out[key] = value
return out
def _decode_numpy(obj: dict[Any, Any]) -> Any:
if obj.get(b"nd") is True and obj.get(b"kind") == b"O":
raise ValueError("object dtype numpy arrays are not supported")
decoded = msgpack_numpy.decode(obj)
if isinstance(decoded, np.ndarray) and decoded.dtype.kind == "O":
raise ValueError("object dtype numpy arrays are not supported")
if isinstance(decoded, dict):
decoded = _normalize_map_keys(decoded)
return decoded
def encode_frame(frame: Frame | dict[str, Any]) -> bytes:
if isinstance(frame, Frame):
wire = frame.to_wire_dict()
else:
wire = dict(frame)
try:
return msgpack.packb(wire, default=_encode_numpy, use_bin_type=True)
except WsError:
raise
except Exception as exc:
raise WsError(ErrorCode.INVALID_FRAME, f"msgpack encode failed: {exc}") from exc
def decode_frame(data: bytes) -> dict[str, Any]:
try:
obj = msgpack.unpackb(
data,
raw=False,
# Observations may key dicts by env index. Accept int keys here;
# _decode_numpy then normalizes them to strings so models see the
# same key type the legacy JSON codec delivered.
strict_map_key=False,
object_hook=_decode_numpy,
)
except Exception as exc:
raise WsError(ErrorCode.INVALID_FRAME, f"msgpack decode failed: {exc}") from exc
if not isinstance(obj, dict):
raise WsError(ErrorCode.INVALID_FRAME, "frame must be a msgpack map")
return obj
def decode_envelope(data: bytes) -> Frame:
try:
return Frame.from_wire_dict(decode_frame(data))
except WsError:
raise
except (TypeError, ValueError, ValidationError) as exc:
raise WsError(
ErrorCode.INVALID_FRAME, f"invalid frame envelope: {exc}"
) from exc

View File

@ -0,0 +1,33 @@
from __future__ import annotations
from enum import Enum
from typing import Any
class ErrorCode(str, Enum):
INVALID_FRAME = "invalid_frame"
UNKNOWN_MESSAGE_TYPE = "unknown_message_type"
TIMEOUT = "timeout"
CALL_FAILED = "call_failed"
INFER_FAILED = "infer_failed"
RESET_FAILED = "reset_failed"
INTERNAL = "internal"
class WsError(Exception):
def __init__(
self, code: ErrorCode, message: str, *, details: dict[str, Any] | None = None
):
super().__init__(message)
self.code = code
self.message = message
self.details = details or {}
class ServerRestartedError(ConnectionError):
"""The policy server process restarted mid-evaluation.
A fresh server holds a fresh model, so continuing would silently mix
pre-restart and post-restart policy state. Raised instead of retrying so
the evaluation fails loudly rather than producing corrupt results.
"""

View File

@ -0,0 +1,33 @@
"""Shared validation for the websocket keepalive knobs.
Both the policy server and the eval client read `ws_ping_interval_s` /
`ws_ping_timeout_s` straight out of `deploy.yml`, so they need the same
answer for what a valid value is.
"""
from __future__ import annotations
import math
from typing import Any
def normalize_ws_ping(value: Any, *, field_name: str) -> float | None:
"""Return the interval in seconds, or None when keepalive is disabled.
`null` is meaningful (it turns keepalive off), so it is accepted; a bool
is not, because YAML's `true` would otherwise silently become 1 second.
"""
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(
f"{field_name} must be null or a positive number, got {value!r}"
)
seconds = float(value)
# nan would silently disable the timeout comparison inside websockets;
# inf would keep the connection open forever. Both are config mistakes.
if not math.isfinite(seconds) or seconds <= 0:
raise ValueError(
f"{field_name} must be null or a positive finite number, got {value!r}"
)
return seconds

View File

@ -0,0 +1,35 @@
"""WebSocket message types."""
from __future__ import annotations
from enum import Enum
class MessageType(str, Enum):
HELLO = "hello"
HELLO_ACK = "hello_ack"
PREPARE_CASE = "prepare_case"
PREPARE_CASE_ACK = "prepare_case_ack"
RESET = "reset"
RESET_RESULT = "reset_result"
CALL = "call"
CALL_RESULT = "call_result"
INFER = "infer"
INFER_RESULT = "infer_result"
TRIAL_END = "trial_end"
TRIAL_END_ACK = "trial_end_ack"
HEARTBEAT = "heartbeat"
HEARTBEAT_ACK = "heartbeat_ack"
CLOSE = "close"
ERROR = "error"
REQUEST_RESPONSE_PAIRS: dict[MessageType, MessageType] = {
MessageType.HELLO: MessageType.HELLO_ACK,
MessageType.PREPARE_CASE: MessageType.PREPARE_CASE_ACK,
MessageType.RESET: MessageType.RESET_RESULT,
MessageType.CALL: MessageType.CALL_RESULT,
MessageType.INFER: MessageType.INFER_RESULT,
MessageType.TRIAL_END: MessageType.TRIAL_END_ACK,
MessageType.HEARTBEAT: MessageType.HEARTBEAT_ACK,
}

View File

@ -0,0 +1,93 @@
from __future__ import annotations
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any
from pydantic import BaseModel, Field
try:
from pydantic import ConfigDict
_PYDANTIC_V2 = True
except ImportError: # pydantic v1
ConfigDict = None # type: ignore[misc, assignment]
_PYDANTIC_V2 = False
from client_server.ws.protocol.exceptions import ErrorCode, WsError
from client_server.ws.protocol.messages import MessageType
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _frame_to_dict(frame: BaseModel, *, exclude: set[str]) -> dict[str, Any]:
if hasattr(frame, "model_dump"):
return frame.model_dump(mode="json", exclude=exclude)
return frame.dict(exclude=exclude) # pydantic v1
class Frame(BaseModel):
"""One WebSocket binary message."""
if _PYDANTIC_V2:
model_config = ConfigDict(populate_by_name=True, extra="forbid")
else:
class Config:
extra = "forbid"
allow_population_by_field_name = True
message_type: MessageType
request_id: str
evaluation_id: str
action_case_id: str | None = None
trial_id: str | None = None
repeat_index: int | None = None
step: int = 0
sent_at: str = Field(default_factory=_utc_now_iso)
payload: dict[str, Any] = Field(default_factory=dict)
def to_wire_dict(self) -> dict[str, Any]:
data = _frame_to_dict(self, exclude={"payload", "request_id"})
data["message_id"] = self.request_id
data["payload"] = self.payload
return data
@classmethod
def from_wire_dict(cls, data: dict[str, Any]) -> Frame:
message_type_value = data.get("message_type")
if message_type_value is None:
raise WsError(ErrorCode.INVALID_FRAME, "missing message_type")
try:
message_type = MessageType(message_type_value)
except ValueError as exc:
raise WsError(
ErrorCode.INVALID_FRAME, f"unknown message_type: {message_type_value!r}"
) from exc
message_id = data.get("message_id")
if message_id is None:
raise WsError(ErrorCode.INVALID_FRAME, "missing message_id")
evaluation_id = data.get("evaluation_id")
if evaluation_id is None:
raise WsError(ErrorCode.INVALID_FRAME, "missing evaluation_id")
payload = data.get("payload") or {}
if not isinstance(payload, Mapping):
raise WsError(ErrorCode.INVALID_FRAME, "payload must be a map")
return cls(
message_type=message_type,
request_id=str(message_id),
evaluation_id=str(evaluation_id),
action_case_id=data.get("action_case_id"),
trial_id=data.get("trial_id"),
repeat_index=data.get("repeat_index"),
step=int(data.get("step", 0)),
sent_at=str(data.get("sent_at") or _utc_now_iso()),
payload=dict(payload),
)

View File

@ -0,0 +1,355 @@
import argparse
import os
import cv2
from client_server.tcp.model_client import ModelClient
import numpy as np
from XPolicyLab.utils.process_data import get_robot_action_dim_info
Batch_Size = 10
class TestEnv:
def __init__(self, deploy_cfg):
self.success_num, self.episode_num = 0, 0
self._stop_check = None
self.deploy_cfg = deploy_cfg
self.episode_step_limit = 20
self.obs_encoded = deploy_cfg.get('obs_encoded', False)
env_cfg_type = deploy_cfg['env_cfg_type']
self.robot_action_dim_info = get_robot_action_dim_info(env_cfg_type)
if deploy_cfg.get("protocol", "ws") == "ws":
from client_server.ws import WsModelClient
policy_server_url = deploy_cfg["policy_server_url"]
if policy_server_url is None:
policy_server_url = f"ws://{deploy_cfg['host']}:{deploy_cfg['port']}"
self.model_client = WsModelClient(
url=policy_server_url,
evaluation_id=deploy_cfg["evaluation_id"],
trial_id=deploy_cfg["trial_id"],
action_case_id=deploy_cfg["action_case_id"],
repeat_index=deploy_cfg["repeat_index"],
ws_ping_interval_s=deploy_cfg.get("ws_ping_interval_s", 20.0),
ws_ping_timeout_s=deploy_cfg.get("ws_ping_timeout_s", 20.0),
# Omitted keys keep the client defaults; a slow-loading policy
# server mainly needs max_connect_attempts / request_timeout_s.
connect_timeout_s=deploy_cfg.get("connect_timeout_s"),
handshake_timeout_s=deploy_cfg.get("handshake_timeout_s"),
request_timeout_s=deploy_cfg.get("request_timeout_s"),
max_connect_attempts=deploy_cfg.get("max_connect_attempts"),
connect_retry_delay_s=deploy_cfg.get("connect_retry_delay_s"),
max_connect_seconds=deploy_cfg.get("max_connect_seconds"),
close_timeout_s=deploy_cfg.get("close_timeout_s"),
)
else:
self.model_client = ModelClient(host=deploy_cfg['host'], port=deploy_cfg['port'])
def set_stop_check(self, stop_check):
self._stop_check = stop_check
def get_obs(self, env_idx=0):
# v1.0
demo_obs = { # aloha
"vision": {
"cam_head": {
"color": np.zeros((480, 640, 3), dtype=np.uint8),
"depth": np.zeros((480, 640, 3), dtype=np.uint8),
"intrinsic_matrix": [
[615.0, 0.0, 320.0],
[0.0, 615.0, 240.0],
[0.0, 0.0, 1.0],
],
"extrinsics_matrix": [
[1.0, 0.0, 0.0, 0.10],
[0.0, 1.0, 0.0, 1.20],
[0.0, 0.0, 1.0, 1.50],
[0.0, 0.0, 0.0, 1.0],
],
"shape": (480, 640),
},
"cam_left_wrist": {
"color": np.zeros((480, 640, 3), dtype=np.uint8),
"depth": np.zeros((480, 640, 3), dtype=np.uint8),
"intrinsic_matrix": [
[615.0, 0.0, 320.0],
[0.0, 615.0, 240.0],
[0.0, 0.0, 1.0],
],
"extrinsics_matrix": [
[1.0, 0.0, 0.0, 0.10],
[0.0, 1.0, 0.0, 1.20],
[0.0, 0.0, 1.0, 1.50],
[0.0, 0.0, 0.0, 1.0],
],
"shape": (480, 640),
},
"cam_right_wrist": {
"color": np.zeros((480, 640, 3), dtype=np.uint8),
"depth": np.zeros((480, 640, 3), dtype=np.uint8),
"intrinsic_matrix": [
[615.0, 0.0, 320.0],
[0.0, 615.0, 240.0],
[0.0, 0.0, 1.0],
],
"extrinsics_matrix": [
[1.0, 0.0, 0.0, 0.10],
[0.0, 1.0, 0.0, 1.20],
[0.0, 0.0, 1.0, 1.50],
[0.0, 0.0, 0.0, 1.0],
],
"shape": (480, 640),
},
},
"instruction": "language instruction",
"state": {
"mobile": {
"base_pose": [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], # x,y,z + quat
"base_twist": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], # vx,vy,vz, wx,wy,wz
},
},
"additional_info": {
"frequency": 30, # Hz
},
"data_format_version": "v1.0",
"env_idx": env_idx
}
state = demo_obs.setdefault("state", {})
arm_dims = self.robot_action_dim_info["arm_dim"]
ee_dims = self.robot_action_dim_info["ee_dim"]
if len(arm_dims) == 1:
prefixes = [""]
elif len(arm_dims) == 2:
prefixes = ["left_", "right_"]
else:
raise ValueError(f"Unsupported arm count: {len(arm_dims)}")
for i, prefix in enumerate(prefixes):
state[f"{prefix}arm_joint_state"] = np.zeros(arm_dims[i], dtype=np.float32)
state[f"{prefix}ee_joint_state"] = np.zeros(ee_dims[i], dtype=np.float32)
state[f"{prefix}ee_pose"] = np.ones(7, dtype=np.float32)
state[f"{prefix}tcp_pose"] = np.zeros(7, dtype=np.float32)
state[f"{prefix}delta_ee_pose"] = np.zeros(7, dtype=np.float32)
if self.obs_encoded:
self._encode_obs_colors(demo_obs)
return demo_obs
@staticmethod
def _encode_obs_colors(obs):
"""Send encoded colors so the server-side decode path gets exercised."""
vision = obs["vision"]
buffer = cv2.imencode(".jpg", vision["cam_head"]["color"])[1].reshape(-1)
vision["cam_head"]["color"] = buffer # 1-D uint8 buffer
vision["cam_left_wrist"]["color"] = buffer.tobytes() # raw bytes
# cam_right_wrist stays a plain array, covering the passthrough branch.
def get_obs_batch(self, env_idx_list):
demo_obs_list = [self.get_obs(env_idx) for env_idx in env_idx_list]
return demo_obs_list
def eval_one_episode(self):
policy_name = self.deploy_cfg['policy_name']
try:
eval_module = __import__(f'XPolicyLab.policy.{policy_name}.deploy', fromlist=['eval_one_episode'])
except ImportError as e:
print("[TestEnv]", f"Failed to import policy module: XPolicyLab.policy.{policy_name}.deploy. Error: {e}", "ERROR")
raise e
if not hasattr(eval_module, 'eval_one_episode'):
print("[TestEnv]", f"Module '.{policy_name}.deploy' does not have 'eval_one_episode' function", "ERROR")
raise AttributeError(f"Missing eval_one_episode in policy module")
eval_module.eval_one_episode(TASK_ENV=self, model_client=self.model_client)
def eval_one_episode_batch(self):
policy_name = self.deploy_cfg['policy_name']
try:
eval_module = __import__(f'XPolicyLab.policy.{policy_name}.deploy', fromlist=['eval_one_episode_batch'])
except ImportError as e:
print("[TestEnv]", f"Failed to import policy module: XPolicyLab.policy.{policy_name}.deploy. Error: {e}", "ERROR")
raise e
if not hasattr(eval_module, 'eval_one_episode_batch'):
print("[TestEnv]", f"Module '.{policy_name}.deploy' does not have 'eval_one_episode_batch' function", "ERROR")
raise AttributeError(f"Missing eval_one_episode_batch in policy module")
eval_module.eval_one_episode_batch(TASK_ENV=self, model_client=self.model_client)
def reset(self):
self.model_client.call(func_name="reset")
self.episode_step = 0
def take_action(self, action):
print(f"[TestEnv] Action Step: {self.episode_step} / {self.episode_step_limit} (step_limit)")
self.episode_step += 1
validate_robot_state_dict(action, self.robot_action_dim_info)
def take_action_batch(self, action_list, env_idx_list):
print(f"[TestEnv] Action Step: {self.episode_step} / {self.episode_step_limit} (step_limit)")
self.episode_step += 1
assert len(action_list) == len(env_idx_list), f"action num != env num: {len(action_list)} != {len(env_idx_list)}"
for action in action_list:
validate_robot_state_dict(action, self.robot_action_dim_info)
def is_episode_end(self):
if self._stop_check is not None and self._stop_check():
print("[TestEnv] Check Episode End: stop requested")
return True
print("[TestEnv] Check Episode End:", self.episode_step >= self.episode_step_limit)
return self.episode_step >= self.episode_step_limit
def finish_episode(self):
print("[TestEnv] Episode finished")
def get_running_env_idx_list(self):
# For demonstration, we assume all envs are running. Replace with actual logic if needed.
return list(range(Batch_Size))
def validate_robot_state_dict(state_dict: dict, robot_action_dim_info: dict) -> None:
"""
Validate whether the state_dict keys use the correct prefixes and dimensions.
Args:
state_dict: e.g. demo_obs["state"]
robot_action_dim_info: {
"arm_dim": [...],
"ee_dim": [...],
}
Raises:
KeyError: if required keys are missing
ValueError: if unexpected keys or wrong dimensions are found
TypeError: if values are not array-like
"""
arm_dims = robot_action_dim_info["arm_dim"]
ee_dims = robot_action_dim_info["ee_dim"]
if len(arm_dims) != len(ee_dims):
raise ValueError(
f"robot_action_dim_info mismatch: len(arm_dim)={len(arm_dims)} "
f"!= len(ee_dim)={len(ee_dims)}"
)
arm_count = len(arm_dims)
if arm_count == 1:
expected = {
"arm_joint_state": arm_dims[0],
"ee_joint_state": ee_dims[0],
"ee_pose": 7,
"tcp_pose": 7,
"delta_ee_pose": 7,
}
forbidden_prefixes = ("left_", "right_")
elif arm_count == 2:
expected = {
"left_arm_joint_state": arm_dims[0],
"left_ee_joint_state": ee_dims[0],
"left_ee_pose": 7,
"left_tcp_pose": 7,
"left_delta_ee_pose": 7,
"right_arm_joint_state": arm_dims[1],
"right_ee_joint_state": ee_dims[1],
"right_ee_pose": 7,
"right_tcp_pose": 7,
"right_delta_ee_pose": 7,
}
forbidden_prefixes = ()
else:
raise ValueError(f"Unsupported arm count: {arm_count}")
if forbidden_prefixes:
bad_prefixed_keys = [
k for k in state_dict.keys()
if k.startswith(forbidden_prefixes)
]
if bad_prefixed_keys:
raise ValueError(
f"Single-arm robot should not contain prefixed keys, "
f"but got: {bad_prefixed_keys}"
)
unexpected_keys = [k for k in state_dict if k not in expected]
if unexpected_keys:
raise ValueError(f"Unexpected state keys: {unexpected_keys}")
for key, expected_dim in expected.items():
if not key in state_dict.keys():
continue
value = state_dict[key]
if not isinstance(value, (np.ndarray, list, tuple)):
raise TypeError(
f"state_dict['{key}'] must be array-like, got {type(value)}"
)
arr = np.asarray(value)
if arr.ndim != 1:
raise ValueError(
f"state_dict['{key}'] must be 1D, got shape {arr.shape}"
)
if arr.shape[0] != expected_dim:
raise ValueError(
f"state_dict['{key}'] dim mismatch: expected {expected_dim}, "
f"got shape {arr.shape}"
)
def str2bool(v):
if isinstance(v, bool):
return v
if v.lower() in ("yes", "true", "t", "1"):
return True
if v.lower() in ("no", "false", "f", "0"):
return False
raise argparse.ArgumentTypeError("Boolean value expected.")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--bench_name", required=True, type=str)
parser.add_argument("--task_name", required=True, type=str)
parser.add_argument("--env_cfg_type", type=str, required=True)
parser.add_argument("--policy_name", type=str, required=True, help="XPolicyLab module name for deployment")
parser.add_argument("--protocol", choices=("legacy_tcp", "ws"), default="ws")
parser.add_argument("--host", type=str, default="localhost", help="server host")
parser.add_argument("--port", type=int, required=True, help="server port")
parser.add_argument("--policy_server_url", type=str)
parser.add_argument("--evaluation_id", type=str, default="debug-eval")
parser.add_argument("--action_case_id", type=str)
parser.add_argument("--trial_id", type=str, default="debug-trial")
parser.add_argument("--repeat_index", type=int)
parser.add_argument("--eval_episode_num", type=int, default=10, help="number of evaluation episodes")
parser.add_argument("--eval_batch", type=str2bool, default=False, help="whether to run batch evaluation")
parser.add_argument("--obs_encoded", type=str2bool, default=os.environ.get("DEBUG_OBS_ENCODED", "0"),
help="send encoded camera colors to exercise the server-side decode path")
args_cli = parser.parse_args()
deploy_cfg = vars(args_cli)
test_env = TestEnv(deploy_cfg)
eval_batch = deploy_cfg['eval_batch']
try:
# Load XPolicyLab
for idx in range(deploy_cfg["eval_episode_num"]):
print(f"\033[94m🚀 Running Episode {idx}\033[0m")
test_env.reset() # reset model, robot, and environment
if not eval_batch:
test_env.eval_one_episode()
else:
test_env.eval_one_episode_batch()
test_env.finish_episode()
finally:
close = getattr(test_env.model_client, "close", None)
if callable(close):
close()

View File

@ -0,0 +1,104 @@
class ModelTemplate:
"""
Base template for a policy/model class.
策略/模型类的基础模板。
This template defines the minimal interface required for:
该模板定义了一个模型在以下场景中常用的最小接口:
1. loading or building a model
加载或构建模型
2. updating observations from the environment
接收并更新环境观测
3. generating actions
生成动作
4. resetting internal states between episodes
在每个 episode 之间重置内部状态
Notes / 说明:
- You should override the methods in your own subclass or implementation.
你应该在自己的子类或具体实现中重写这些方法。
- `self.model` can be a neural network, a planner, or any policy object.
`self.model` 可以是神经网络、规划器,或者任意策略对象。
"""
RED = "\033[31m"
RESET = "\033[0m"
def __init__(self):
"""
Initialize the template model.
初始化模板模型。
Attributes / 属性:
self.model:
The actual model/policy object.
实际的模型/策略对象。
"""
self.model = None
def _error_msg(self, msg: str) -> str:
return f"{self.RED}[ERROR] {msg}{self.RESET}"
def update_obs(self, obs):
"""
Update the current observation used by the model.
更新当前模型使用的观测。
The policy server decodes camera colors before calling this, so
obs["vision"][<camera>]["color"] is always an image array.
策略服务端已完成图像解码,此处拿到的 color 一定是图像数组。
"""
raise NotImplementedError(
self._error_msg("update_obs() must be implemented by the user.")
)
def update_obs_batch(self, obs_list):
"""
Update the current observation used by the model.
更新当前模型使用的观测。
Same decoding guarantee as update_obs().
与 update_obs() 一样,图像已由服务端解码。
"""
raise NotImplementedError(
self._error_msg("update_obs_batch() must be implemented by the user.")
)
def get_action(self):
"""
Predict or generate an action from the current observation/state.
根据当前观测或内部状态预测/生成动作。
"""
raise NotImplementedError(
self._error_msg("get_action() must be implemented by the user.")
)
def get_action_batch(self, env_idx_list=None):
"""
Predict or generate an action from the current observation/state.
根据当前观测或内部状态预测/生成动作。
"""
raise NotImplementedError(
self._error_msg("get_action_batch() must be implemented by the user.")
)
def reset(self):
"""
Reset the internal state of the model at the start of a new episode.
在新 episode 开始时重置模型内部状态。
"""
pass
def prepare_case(self, case_meta=None):
"""
Optional hook called before an action case starts.
在 action case 开始前调用的 hook。
"""
pass
def on_trial_end(self, result=None):
"""
Optional hook called after a trial finishes.
在单次 trial 结束后调用的 hook。
"""
pass

View File

@ -0,0 +1,26 @@
[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"
[project]
name = "xpolicylab"
version = "0.0.1"
requires-python = ">=3.10"
dependencies = [
"numpy>=1.23",
"pyyaml>=6",
"opencv-python-headless>=4.8",
"h5py>=3.8",
"websockets>=14.0",
"msgpack>=1.0.8",
"msgpack-numpy>=0.4.8",
"pydantic>=2.5",
]
[tool.setuptools]
py-modules = ["XPolicyLab"]
[tool.setuptools.packages.find]
where = ["."]
include = ["client_server*", "utils*"]
exclude = ["policy*", "build*", "scripts*", "__pycache__*", "tests*"]

View File

@ -0,0 +1,236 @@
#!/usr/bin/env bash
# Download the RoboDojo dataset from Hugging Face to XPolicyLab/data/
#
# Usage:
# bash scripts/RoboDojo/download_robodojo_data.sh <type>
#
# Example:
# bash scripts/RoboDojo/download_robodojo_data.sh hdf5
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # XPolicyLab root directory
DATA_ROOT="${PROJECT_ROOT}/../data"
DATA_TYPE="${1:-}" # Data format: lerobot_v3.0 / lerobot_v2.1 / hdf5 / demo / real
HF_REPO_ID="${HF_REPO_ID:-RoboDojo-Benchmark/Robodojo}"
HF_REVISION="${HF_REVISION:-main}"
HF_MAX_WORKERS="${HF_MAX_WORKERS:-16}"
HF_RETRY_WAIT="${HF_RETRY_WAIT:-60}"
HF_MAX_RETRIES="${HF_MAX_RETRIES:-10}"
HF_DOWNLOAD_TIMEOUT="${HF_DOWNLOAD_TIMEOUT:-300}"
usage() {
cat <<'EOF'
Usage: bash scripts/RoboDojo/download_robodojo_data.sh <type>
Types:
lerobot_v3.0 -> RoboDojo_lerobot_v30_video
lerobot_v2.1 -> RoboDojo_lerobot_v21_video
hdf5 -> RoboDojo
demo -> demo
real -> RoboDojo_real
Example:
bash scripts/RoboDojo/download_robodojo_data.sh hdf5
bash scripts/RoboDojo/download_robodojo_data.sh real
Environment:
HF_REPO_ID default: RoboDojo-Benchmark/Robodojo
HF_REVISION default: main
HF_MAX_WORKERS parallel file downloads (default: 16)
HF_RETRY_WAIT seconds to wait before retry (default: 60)
HF_MAX_RETRIES max retry attempts (default: 10)
HF_DOWNLOAD_TIMEOUT per-file timeout in seconds (default: 300)
HF_FORCE_DOWNLOAD set to 1 to re-download even if complete
EOF
}
if [[ -z "${DATA_TYPE}" ]]; then
usage
exit 1
fi
mkdir -p "${DATA_ROOT}"
resolve_data_paths() {
case "${DATA_TYPE}" in
lerobot_v3.0)
DATA_DIR_NAME="RoboDojo_lerobot_v30_video"
;;
lerobot_v2.1)
DATA_DIR_NAME="RoboDojo_lerobot_v21_video"
;;
hdf5)
DATA_DIR_NAME="RoboDojo"
;;
demo)
DATA_DIR_NAME="demo"
;;
real|RoboDojo_real)
DATA_DIR_NAME="RoboDojo_real"
;;
*)
echo "Invalid type: ${DATA_TYPE}" >&2
return 1
;;
esac
REMOTE_DIR="data/${DATA_DIR_NAME}"
TARGET_DIR="${DATA_ROOT}/${DATA_DIR_NAME}"
}
ensure_hf_deps() {
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 not found" >&2
exit 1
fi
python3 - <<'PY'
import importlib.util
import subprocess
import sys
missing = [
name for name in ("huggingface_hub", "hf_transfer")
if importlib.util.find_spec(name) is None
]
if missing:
subprocess.check_call([
sys.executable,
"-m",
"pip",
"install",
"-U",
"huggingface_hub",
"hf_transfer",
])
PY
}
# HuggingFace: snapshot_download + allow_patterns + hf_transfer parallel batch download for large files
download_hf_folder() {
local repo_id="$1"
local remote_dir="$2"
local target_dir="$3"
local complete_marker="${target_dir}/.download_complete"
if [[ -f "${complete_marker}" && "${HF_FORCE_DOWNLOAD:-0}" != "1" ]]; then
echo "==> Target already exists, skip: ${target_dir}"
return 0
fi
if [[ -d "${target_dir}" && "${HF_FORCE_DOWNLOAD:-0}" != "1" ]]; then
echo "==> Resuming partial download: ${target_dir}"
elif [[ -d "${target_dir}" ]]; then
echo "==> Force re-download: ${target_dir}"
rm -rf "${target_dir}"
fi
ensure_hf_deps
# hf_transfer must be enabled before importing huggingface_hub
export HF_HUB_ENABLE_HF_TRANSFER=1
export HF_HUB_DOWNLOAD_TIMEOUT="${HF_DOWNLOAD_TIMEOUT}"
export HF_TRANSFER_MAX_CONCURRENT_DOWNLOADS="${HF_MAX_WORKERS}"
echo "==> Downloading ${remote_dir} from HuggingFace ${repo_id}"
echo "==> Parallel workers: ${HF_MAX_WORKERS}, timeout: ${HF_DOWNLOAD_TIMEOUT}s"
HF_REPO_ID="${repo_id}" \
HF_REVISION="${HF_REVISION}" \
REMOTE_DIR="${remote_dir}" \
DATA_ROOT="${DATA_ROOT}" \
TARGET_DIR="${target_dir}" \
HF_MAX_WORKERS="${HF_MAX_WORKERS}" \
HF_RETRY_WAIT="${HF_RETRY_WAIT}" \
HF_MAX_RETRIES="${HF_MAX_RETRIES}" \
python3 - <<'PY'
import os
import sys
import time
from pathlib import Path
from huggingface_hub import snapshot_download
from huggingface_hub.utils import HfHubHTTPError
repo_id = os.environ["HF_REPO_ID"]
revision = os.environ["HF_REVISION"]
remote_dir = os.environ["REMOTE_DIR"]
data_root = Path(os.environ["DATA_ROOT"])
target_dir = Path(os.environ["TARGET_DIR"])
complete_marker = target_dir / ".download_complete"
max_workers = int(os.environ["HF_MAX_WORKERS"])
retry_wait = int(os.environ["HF_RETRY_WAIT"])
max_retries = int(os.environ["HF_MAX_RETRIES"])
data_root.mkdir(parents=True, exist_ok=True)
download_root = data_root.parent
def is_retryable(exc: BaseException) -> bool:
err = str(exc).lower()
if isinstance(exc, HfHubHTTPError) and exc.response is not None:
if exc.response.status_code in (408, 429, 500, 502, 503, 504):
return True
return any(
token in err
for token in (
"429",
"too many requests",
"rate limit",
"timeout",
"timed out",
"connection reset",
"connection aborted",
"temporary failure",
"503",
"502",
)
)
attempt = 0
while True:
attempt += 1
try:
print(f"==> Download attempt {attempt}/{max_retries} (max_workers={max_workers})")
snapshot_download(
repo_id=repo_id,
repo_type="dataset",
revision=revision,
local_dir=str(download_root),
allow_patterns=[f"{remote_dir}/**"],
max_workers=max_workers,
)
break
except Exception as exc:
if attempt >= max_retries or not is_retryable(exc):
raise
print(
f"==> Transient error, retry in {retry_wait}s: {exc}",
file=sys.stderr,
)
time.sleep(retry_wait)
if not target_dir.is_dir():
print(f"Remote folder not found: {remote_dir}", file=sys.stderr)
sys.exit(1)
complete_marker.write_text(f"repo_id={repo_id}\nrevision={revision}\n")
print(f"==> Saved to {target_dir}")
PY
if [[ ! -d "${target_dir}" ]]; then
echo "Remote folder not found: ${remote_dir}" >&2
exit 1
fi
echo "==> Download complete: ${target_dir}"
}
if ! resolve_data_paths; then
usage
exit 1
fi
download_hf_folder "${HF_REPO_ID}" "${REMOTE_DIR}" "${TARGET_DIR}"

View File

@ -0,0 +1,3 @@
policy_name=${1}
cp -r policy/demo_policy policy/${policy_name}

View File

@ -0,0 +1,159 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
DOWNLOAD_ROOT="${PROJECT_ROOT}/.hf_download_cache/robodojo_demo"
REPO_ID="${HF_REPO_ID:-DaMiTian/RoboDojo_demo_data}"
REMOTE_SUBDIR="${HF_REMOTE_SUBDIR:-archives/robodojo_tmp}"
HF_REVISION="${HF_REVISION:-main}"
DATA_ROOT="${PROJECT_ROOT}/data"
DATASET_NAME="${ROBODOJO_DEMO_DATASET:-RoboDojo_demo}"
TARGET_DATA_DIR="${DATA_ROOT}/${DATASET_NAME}"
TARGET_ENV_CFG_DIR="${PROJECT_ROOT}/env_cfg"
echo "==> Project root: ${PROJECT_ROOT}"
echo "==> Demo data dir: data/${DATASET_NAME}"
echo "==> Repo: ${REPO_ID}"
echo "==> Remote subdir: ${REMOTE_SUBDIR}"
mkdir -p "${DOWNLOAD_ROOT}"
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 未找到" >&2
exit 1
fi
echo "==> Ensuring Python dependencies"
python3 - <<'PY'
import importlib.util
import subprocess
import sys
missing = [
name for name in ("huggingface_hub", "hf_transfer")
if importlib.util.find_spec(name) is None
]
if missing:
subprocess.check_call([
sys.executable,
"-m",
"pip",
"install",
"-U",
"huggingface_hub",
"hf_transfer",
])
PY
echo "==> Downloading archive parts from Hugging Face"
REPO_ID="${REPO_ID}" REMOTE_SUBDIR="${REMOTE_SUBDIR}" HF_REVISION="${HF_REVISION}" DOWNLOAD_ROOT="${DOWNLOAD_ROOT}" python3 - <<'PY'
import os
from pathlib import Path
from huggingface_hub import snapshot_download
repo_id = os.environ["REPO_ID"]
remote_subdir = os.environ["REMOTE_SUBDIR"].strip("/")
revision = os.environ["HF_REVISION"]
download_root = Path(os.environ["DOWNLOAD_ROOT"])
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
patterns = [f"{remote_subdir}/*"] if remote_subdir else ["*"]
snapshot_download(
repo_id=repo_id,
repo_type="dataset",
revision=revision,
local_dir=str(download_root),
local_dir_use_symlinks=False,
resume_download=True,
allow_patterns=patterns,
)
print(f"Downloaded to: {download_root}")
PY
ARCHIVE_DIR="${DOWNLOAD_ROOT}/${REMOTE_SUBDIR}"
if [[ ! -d "${ARCHIVE_DIR}" ]]; then
echo "未找到下载目录: ${ARCHIVE_DIR}" >&2
exit 1
fi
echo "==> Verifying downloaded files"
if [[ -f "${ARCHIVE_DIR}/SHA256SUMS" ]]; then
(
cd "${ARCHIVE_DIR}"
sha256sum -c SHA256SUMS
)
else
echo "警告: 未发现 SHA256SUMS,跳过校验"
fi
shopt -s nullglob
parts=("${ARCHIVE_DIR}"/*.part-*)
shopt -u nullglob
if [[ ${#parts[@]} -eq 0 ]]; then
echo "未找到分片文件: ${ARCHIVE_DIR}/*.part-*" >&2
exit 1
fi
first_part="$(basename "${parts[0]}")"
archive_name="${first_part%.part-*}"
archive_path="${ARCHIVE_DIR}/${archive_name}"
extract_root="${DOWNLOAD_ROOT}/extracted"
echo "==> Reassembling archive: ${archive_name}"
cat "${ARCHIVE_DIR}"/*.part-* > "${archive_path}"
rm -rf "${extract_root}"
mkdir -p "${extract_root}"
echo "==> Extracting selected paths"
case "${archive_name}" in
*.tar.zst)
tar -I zstd -xf "${archive_path}" -C "${extract_root}" tmp/data tmp/env_cfg
;;
*.tar.gz)
tar -xzf "${archive_path}" -C "${extract_root}" tmp/data tmp/env_cfg
;;
*.tar)
tar -xf "${archive_path}" -C "${extract_root}" tmp/data tmp/env_cfg
;;
*)
echo "不支持的压缩格式: ${archive_name}" >&2
exit 1
;;
esac
if [[ ! -d "${extract_root}/tmp/data" || ! -d "${extract_root}/tmp/env_cfg" ]]; then
echo "压缩包内未找到 tmp/data 或 tmp/env_cfg" >&2
exit 1
fi
echo "==> Restoring demo data -> data/${DATASET_NAME} (not data/RoboDojo)"
src_data="${extract_root}/tmp/data"
mkdir -p "${DATA_ROOT}"
rm -rf "${TARGET_DATA_DIR}"
# Archives often contain tmp/data/RoboDojo/...; install uniformly to data/RoboDojo_demo/ to avoid conflicts with the full RoboDojo dataset
if [[ -d "${src_data}/${DATASET_NAME}" ]]; then
mv "${src_data}/${DATASET_NAME}" "${TARGET_DATA_DIR}"
elif [[ -d "${src_data}/RoboDojo" ]]; then
mv "${src_data}/RoboDojo" "${TARGET_DATA_DIR}"
else
mv "${src_data}" "${TARGET_DATA_DIR}"
fi
if [[ -e "${TARGET_ENV_CFG_DIR}" ]]; then
echo "env_cfg 已存在,为避免覆盖请先手动处理: ${TARGET_ENV_CFG_DIR}" >&2
exit 1
fi
mv "${extract_root}/tmp/env_cfg" "${TARGET_ENV_CFG_DIR}"
echo "==> Done"
du -sh "${TARGET_DATA_DIR}" "${TARGET_ENV_CFG_DIR}"

View File

@ -0,0 +1,314 @@
import argparse
import sys
from pathlib import Path
import h5py
import numpy as np
from tqdm import tqdm
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from XPolicyLab.utils.data_loader import load
from XPolicyLab.utils.process_data import decode_image_bit
CAMERA_CANDIDATES = {
"cam_high": [
("vision", "cam_head", "colors"),
("vision", "cam_head", "color"),
("cam_head", "color"),
("cam_head", "colors"),
("cam_high", "color"),
("cam_high", "colors"),
],
"cam_left_wrist": [
("vision", "cam_left_wrist", "colors"),
("vision", "cam_left_wrist", "color"),
("cam_left_wrist", "color"),
("cam_left_wrist", "colors"),
],
"cam_right_wrist": [
("vision", "cam_right_wrist", "colors"),
("vision", "cam_right_wrist", "color"),
("cam_right_wrist", "color"),
("cam_right_wrist", "colors"),
],
"cam_wrist": [
("vision", "cam_wrist", "colors"),
("vision", "cam_wrist", "color"),
("cam_wrist", "color"),
("cam_wrist", "colors"),
],
}
def _get_nested(data, *keys, default=None):
cur = data
for key in keys:
if not isinstance(cur, dict) or key not in cur:
return default
cur = cur[key]
return cur
def _ensure_2d_float32(array, name):
arr = np.asarray(array, dtype=np.float32)
if arr.ndim == 1:
arr = arr[:, None]
if arr.ndim != 2:
raise ValueError(f"{name} should be 2D, got shape {arr.shape}")
return arr
def _ensure_utf8_strings(value):
if value is None:
return []
if isinstance(value, str):
return [value]
if isinstance(value, bytes):
return [value.decode("utf-8")]
if isinstance(value, np.ndarray):
if value.ndim == 0:
return _ensure_utf8_strings(value.item())
return [str(x.decode("utf-8") if isinstance(x, (bytes, np.bytes_)) else x) for x in value.tolist()]
if isinstance(value, (list, tuple)):
result = []
for item in value:
result.extend(_ensure_utf8_strings(item))
return result
return [str(value)]
def _find_instructions(data):
candidates = [
_get_nested(data, "instructions"),
_get_nested(data, "instruction"),
_get_nested(data, "extra_episode_info", "instructions"),
_get_nested(data, "extra_episode_info", "instruction"),
_get_nested(data, "meta", "instructions"),
_get_nested(data, "metadata", "instructions"),
]
for candidate in candidates:
strings = [s for s in _ensure_utf8_strings(candidate) if s]
if strings:
return strings
return []
def _find_camera_array(data, camera_name):
for keys in CAMERA_CANDIDATES[camera_name]:
value = _get_nested(data, *keys)
if value is not None:
return _decode_images_if_needed(value)
return None
def _decode_images_if_needed(images):
frames = np.asarray(decode_image_bit(images))
if frames.ndim == 3:
frames = frames[None, ...]
if frames.ndim != 4:
raise ValueError(f"Expected decoded frames with shape [T,H,W,3], got {frames.shape}")
return frames.astype(np.uint8, copy=False)
def _concat_state_parts(parts, name):
valid_parts = []
horizon = None
for part_name, value in parts:
if value is None:
continue
arr = _ensure_2d_float32(value, f"{name}.{part_name}")
if horizon is None:
horizon = arr.shape[0]
elif arr.shape[0] != horizon:
raise ValueError(
f"{name}.{part_name} horizon mismatch: expected {horizon}, got {arr.shape[0]}"
)
valid_parts.append(arr)
if not valid_parts:
return None
return np.concatenate(valid_parts, axis=1)
def _extract_qpos(data):
qpos = _get_nested(data, "qpos")
if qpos is not None:
return _ensure_2d_float32(qpos, "qpos")
state_qpos = _concat_state_parts(
[
("left_arm_joint_states", _get_nested(data, "state", "left_arm_joint_states")),
("left_ee_joint_states", _get_nested(data, "state", "left_ee_joint_states")),
("right_arm_joint_states", _get_nested(data, "state", "right_arm_joint_states")),
("right_ee_joint_states", _get_nested(data, "state", "right_ee_joint_states")),
],
"state",
)
if state_qpos is not None:
return state_qpos
left_state = _concat_state_parts(
[
("joint", _get_nested(data, "left_arm", "joint")),
("gripper", _get_nested(data, "left_arm", "gripper")),
],
"left_arm",
)
right_state = _concat_state_parts(
[
("joint", _get_nested(data, "right_arm", "joint")),
("gripper", _get_nested(data, "right_arm", "gripper")),
],
"right_arm",
)
parts = [part for part in [left_state, right_state] if part is not None]
if parts:
horizon = parts[0].shape[0]
if any(part.shape[0] != horizon for part in parts):
raise ValueError("left/right arm qpos horizon mismatch")
return np.concatenate(parts, axis=1)
raise ValueError("Cannot find qpos data")
def _extract_action(data, qpos):
action = _get_nested(data, "action")
if action is not None:
action = _ensure_2d_float32(action, "action")
if action.shape[0] != qpos.shape[0]:
raise ValueError(
f"action horizon mismatch: action={action.shape[0]}, qpos={qpos.shape[0]}"
)
return action
left_action = _concat_state_parts(
[
("joint", _get_nested(data, "left_arm", "joint_action")),
("gripper", _get_nested(data, "left_arm", "gripper_action")),
],
"left_arm_action",
)
right_action = _concat_state_parts(
[
("joint", _get_nested(data, "right_arm", "joint_action")),
("gripper", _get_nested(data, "right_arm", "gripper_action")),
],
"right_arm_action",
)
parts = [part for part in [left_action, right_action] if part is not None]
if parts:
horizon = parts[0].shape[0]
if any(part.shape[0] != horizon for part in parts):
raise ValueError("left/right arm action horizon mismatch")
action = np.concatenate(parts, axis=1)
if action.shape[0] != qpos.shape[0]:
raise ValueError(
f"action horizon mismatch: action={action.shape[0]}, qpos={qpos.shape[0]}"
)
return action.astype(np.float32)
state_action = _concat_state_parts(
[
("left_arm_joint_states", _get_nested(data, "state", "left_arm_joint_states")),
("left_ee_joint_states", _get_nested(data, "state", "left_ee_joint_states")),
("right_arm_joint_states", _get_nested(data, "state", "right_arm_joint_states")),
("right_ee_joint_states", _get_nested(data, "state", "right_ee_joint_states")),
],
"state_action",
)
if state_action is not None:
qpos = state_action.astype(np.float32)
if len(qpos) == 1:
return np.zeros_like(qpos, dtype=np.float32)
next_qpos = qpos[1:]
last_action = np.zeros((1, qpos.shape[1]), dtype=np.float32)
return np.concatenate([next_qpos, last_action], axis=0).astype(np.float32)
def _infer_arm_dims(qpos):
qpos_dim = qpos.shape[1]
if qpos_dim % 2 == 0:
return qpos_dim // 2, qpos_dim // 2
return qpos_dim, 0
def convert_one(input_path, output_path, data_type, data_version):
data = load(str(input_path), data_type=data_type, data_version=data_version)
qpos = _extract_qpos(data)
action = _extract_action(data, qpos)
instructions = _find_instructions(data)
left_arm_dim, right_arm_dim = _infer_arm_dims(qpos)
with h5py.File(output_path, "w") as f:
f.create_dataset("action", data=action, dtype=np.float32)
obs = f.create_group("observations")
obs.create_dataset("qpos", data=qpos, dtype=np.float32)
obs.create_dataset("left_arm_dim", data=np.array(left_arm_dim, dtype=np.int32))
obs.create_dataset("right_arm_dim", data=np.array(right_arm_dim, dtype=np.int32))
images_group = obs.create_group("images")
for camera_name in CAMERA_CANDIDATES:
images = _find_camera_array(data, camera_name)
if images is not None:
images_group.create_dataset(camera_name, data=images, dtype=np.uint8)
if instructions:
string_dtype = h5py.string_dtype(encoding="utf-8")
f.create_dataset("instructions", data=np.asarray(instructions, dtype=object), dtype=string_dtype)
def find_input_files(input_dir):
input_dir = Path(input_dir)
files = sorted(input_dir.rglob("*.hdf5"))
files.extend(sorted(input_dir.rglob("*.h5")))
unique_files = []
seen = set()
for file_path in files:
resolved = str(file_path.resolve())
if resolved not in seen:
seen.add(resolved)
unique_files.append(file_path)
return unique_files
def main():
parser = argparse.ArgumentParser(description="Transform dataset to aloha hdf5 format.")
parser.add_argument("input_dir", type=str, help="Input dataset directory")
parser.add_argument("output_dir", type=str, help="Output directory")
parser.add_argument("--data_type", type=str, default="xspark", help="Dataset type, e.g. xspark")
parser.add_argument("--data_version", type=str, default="v1.0", help="Dataset version, e.g. v1.0")
args = parser.parse_args()
input_files = find_input_files(args.input_dir)
if not input_files:
raise FileNotFoundError(f"No .hdf5 or .h5 files found under {args.input_dir}")
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
failures = []
for episode_idx, input_path in enumerate(tqdm(input_files, desc="Converting")):
output_path = output_dir / f"episode_{episode_idx}.hdf5"
try:
convert_one(input_path, output_path, args.data_type, args.data_version)
except Exception as exc:
failures.append((str(input_path), str(exc)))
print(f"Converted {len(input_files) - len(failures)}/{len(input_files)} files to {output_dir}")
if failures:
print("Failed files:")
for file_path, reason in failures:
print(f" - {file_path}: {reason}")
if __name__ == "__main__":
main()

View File

@ -0,0 +1,683 @@
import argparse
import dataclasses
import fnmatch
import random
import sys
from pathlib import Path
from typing import Literal
import cv2
import h5py
import numpy as np
from tqdm import tqdm
import shutil
from lerobot.datasets.lerobot_dataset import HF_LEROBOT_HOME, LeRobotDataset
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from XPolicyLab.utils.data_loader import load
from XPolicyLab.utils.load_file import load_json, load_yaml
from XPolicyLab.utils.process_data import decode_image_bit
DEFAULT_DATASET_NAME = "RoboDojo"
DATA_ROOT = PROJECT_ROOT / "data"
ENV_CFG_ROOT = PROJECT_ROOT / "env_cfg"
ROBOT_INFO_PATH = ENV_CFG_ROOT / "robot" / "_robot_info.json"
DEFAULT_IMAGE_WIDTH = 640
DEFAULT_IMAGE_HEIGHT = 480
# Common source resolutions: 480x640 (RoboDojo) and 240x320 (RoboTwin).
SUPPORTED_IMAGE_SIZES = ((480, 640), (240, 320))
CAMERA_CANDIDATES = {
"cam_high": [
("vision", "cam_head", "colors"),
],
"cam_left_wrist": [
("vision", "cam_left_wrist", "colors"),
],
"cam_right_wrist": [
("vision", "cam_right_wrist", "colors"),
],
}
@dataclasses.dataclass(frozen=True)
class DatasetConfig:
use_videos: bool = True
tolerance_s: float = 0.0001
# Conservative defaults reduce RAM pressure during long conversions.
image_writer_processes: int = 0
image_writer_threads: int = 4
video_backend: str | None = None
DEFAULT_DATASET_CONFIG = DatasetConfig()
def _load_env_metadata(env_cfg_type):
env_cfg = load_yaml(str(ENV_CFG_ROOT / f"{env_cfg_type}.yml"))
robot_name = env_cfg["config"]["robot"]
robot_action_dim_info = load_json(str(ROBOT_INFO_PATH))[robot_name]
fps = env_cfg.get("observation", {}).get("collect_freq", 50)
return robot_name, robot_action_dim_info, fps
def _split_pattern(pattern):
parts = pattern.split(".")
if len(parts) != 3:
raise ValueError(f"Pattern should be <dataset>.<task>.<env_cfg>, got: {pattern}")
return parts[0], parts[1], parts[2]
def _match_name(name, pattern):
return pattern == "*" or fnmatch.fnmatchcase(name, pattern)
def _discover_conversion_targets(patterns):
targets = []
seen = set()
for pattern in patterns:
dataset_pattern, task_pattern, env_pattern = _split_pattern(pattern)
if not DATA_ROOT.exists():
break
for dataset_dir in sorted(path for path in DATA_ROOT.iterdir() if path.is_dir()):
if not _match_name(dataset_dir.name, dataset_pattern):
continue
for task_dir in sorted(path for path in dataset_dir.iterdir() if path.is_dir()):
if not _match_name(task_dir.name, task_pattern):
continue
for env_dir in sorted(path for path in task_dir.iterdir() if path.is_dir()):
if not _match_name(env_dir.name, env_pattern):
continue
input_dir = env_dir / "data"
if not input_dir.is_dir():
continue
target = (dataset_dir.name, task_dir.name, env_dir.name)
if target not in seen:
seen.add(target)
targets.append(target)
return targets
def _dims_from_robot_action_info(robot_action_dim_info):
arm_dims = robot_action_dim_info.get("arm_dim", [])
ee_dims = robot_action_dim_info.get("ee_dim", [])
per_arm_dims = []
for index, arm_dim in enumerate(arm_dims):
ee_dim = ee_dims[index] if index < len(ee_dims) else 0
per_arm_dims.append(arm_dim + ee_dim)
return per_arm_dims
def _plan_target_metadata(targets):
max_per_arm_dims = []
metadata = {}
max_fps = 0
for bench_name, task_name, env_cfg_type in targets:
robot_name, robot_action_dim_info, fps = _load_env_metadata(env_cfg_type)
per_arm_dims = _dims_from_robot_action_info(robot_action_dim_info)
if len(max_per_arm_dims) < len(per_arm_dims):
max_per_arm_dims.extend([0] * (len(per_arm_dims) - len(max_per_arm_dims)))
for index, dim in enumerate(per_arm_dims):
max_per_arm_dims[index] = max(max_per_arm_dims[index], dim)
max_fps = max(max_fps, fps)
metadata[(bench_name, task_name, env_cfg_type)] = {
"robot_name": robot_name,
"robot_action_dim_info": robot_action_dim_info,
"fps": fps,
"per_arm_dims": per_arm_dims,
}
return metadata, max_per_arm_dims, max_fps
def _build_motor_names(robot_action_dim_info):
arm_dims = robot_action_dim_info.get("arm_dim", [])
ee_dims = robot_action_dim_info.get("ee_dim", [])
if not arm_dims:
raise ValueError("robot_action_dim_info.arm_dim is empty")
if len(arm_dims) == 1:
prefixes = ["arm"]
elif len(arm_dims) == 2:
prefixes = ["left", "right"]
else:
prefixes = [f"arm_{index}" for index in range(len(arm_dims))]
motors = []
for index, prefix in enumerate(prefixes):
for joint_idx in range(arm_dims[index]):
motors.append(f"{prefix}_arm_joint_{joint_idx}")
ee_dim = ee_dims[index] if index < len(ee_dims) else 0
if ee_dim == 1:
motors.append(f"{prefix}_gripper")
else:
for ee_idx in range(ee_dim):
motors.append(f"{prefix}_ee_joint_{ee_idx}")
return motors
def _build_motor_names_from_dims(per_arm_dims):
if not per_arm_dims:
raise ValueError("per_arm_dims is empty")
if len(per_arm_dims) == 1:
prefixes = ["arm"]
elif len(per_arm_dims) == 2:
prefixes = ["left", "right"]
else:
prefixes = [f"arm_{index}" for index in range(len(per_arm_dims))]
motors = []
for index, total_dim in enumerate(per_arm_dims):
prefix = prefixes[index]
for joint_idx in range(total_dim):
motors.append(f"{prefix}_joint_{joint_idx}")
return motors
def _expected_state_dim(robot_action_dim_info):
return sum(robot_action_dim_info.get("arm_dim", [])) + sum(robot_action_dim_info.get("ee_dim", []))
def _pad_state_to_target_dims(array, current_dims, target_dims, name):
arr = _ensure_2d_float32(array, name)
expected_dim = sum(current_dims)
if arr.shape[1] != expected_dim:
raise ValueError(f"{name} dim mismatch: expected {expected_dim}, got {arr.shape[1]}")
padded_parts = []
start = 0
for arm_index, target_dim in enumerate(target_dims):
current_dim = current_dims[arm_index] if arm_index < len(current_dims) else 0
end = start + current_dim
part = arr[:, start:end] if current_dim else np.zeros((arr.shape[0], 0), dtype=np.float32)
start = end
if target_dim < current_dim:
raise ValueError(
f"{name} target dim for arm {arm_index} is smaller than current dim: {target_dim} < {current_dim}"
)
if target_dim > current_dim:
padding = np.zeros((arr.shape[0], target_dim - current_dim), dtype=np.float32)
part = np.concatenate([part, padding], axis=1)
padded_parts.append(part)
if start != arr.shape[1]:
raise ValueError(f"{name} dim mismatch after padding: consumed {start}, actual {arr.shape[1]}")
return np.concatenate(padded_parts, axis=1) if padded_parts else arr
def _resolve_input_dir(bench_name, task_name, env_cfg_type, input_dir=None):
if input_dir is not None:
return Path(input_dir)
return DATA_ROOT / bench_name / task_name / env_cfg_type / "data"
def _default_repo_id(bench_name, task_name, env_cfg_type):
return f"{bench_name}_{task_name}_{env_cfg_type}".replace("/", "_").lower()
def create_empty_dataset(
repo_id: str,
robot_type: str,
motors: list[str],
fps: int,
mode: Literal["video", "image"] = "video",
*,
image_height: int = DEFAULT_IMAGE_HEIGHT,
image_width: int = DEFAULT_IMAGE_WIDTH,
dataset_config: DatasetConfig = DEFAULT_DATASET_CONFIG,
) -> LeRobotDataset:
features = {
"observation.state": {
"dtype": "float32",
"shape": (len(motors),),
"names": [
motors,
],
},
"action": {
"dtype": "float32",
"shape": (len(motors),),
"names": [
motors,
],
},
}
for cam in CAMERA_CANDIDATES.keys():
features[f"observation.images.{cam}"] = {
"dtype": mode,
"shape": (3, image_height, image_width),
"names": [
"channels",
"height",
"width",
],
}
if Path(HF_LEROBOT_HOME / repo_id).exists():
shutil.rmtree(HF_LEROBOT_HOME / repo_id)
return LeRobotDataset.create(
repo_id=repo_id,
fps=fps,
robot_type=robot_type,
features=features,
use_videos=dataset_config.use_videos,
tolerance_s=dataset_config.tolerance_s,
image_writer_processes=dataset_config.image_writer_processes,
image_writer_threads=dataset_config.image_writer_threads,
video_backend=dataset_config.video_backend,
)
def _get_nested(data, *keys, default=None):
cur = data
for key in keys:
if not isinstance(cur, dict) or key not in cur:
return default
cur = cur[key]
return cur
def _ensure_2d_float32(array, name):
arr = np.asarray(array, dtype=np.float32)
if arr.ndim == 1:
arr = arr[:, None]
if arr.ndim != 2:
raise ValueError(f"{name} should be 2D, got shape {arr.shape}")
return arr
def _ensure_utf8_strings(value):
if value is None:
return []
if isinstance(value, str):
return [value]
if isinstance(value, bytes):
return [value.decode("utf-8")]
if isinstance(value, np.ndarray):
if value.ndim == 0:
return _ensure_utf8_strings(value.item())
return [str(x.decode("utf-8") if isinstance(x, (bytes, np.bytes_)) else x) for x in value.tolist()]
if isinstance(value, (list, tuple)):
result = []
for item in value:
result.extend(_ensure_utf8_strings(item))
return result
return [str(value)]
def _find_instructions(data):
candidates = [
_get_nested(data, "instructions"),
_get_nested(data, "instruction"),
]
for candidate in candidates:
strings = [s for s in _ensure_utf8_strings(candidate) if s]
if strings:
return strings
return []
def _choose_instruction(data):
instructions = _find_instructions(data)
if not instructions:
return ""
return random.choice(instructions)
def _find_camera_array(data, camera_name, image_height, image_width):
for keys in CAMERA_CANDIDATES[camera_name]:
value = _get_nested(data, *keys)
if value is not None:
return _decode_images_if_needed(value, image_height, image_width)
return None
def _resize_image(image, image_height, image_width):
if image.shape[:2] == (image_height, image_width):
return image
return cv2.resize(image, (image_width, image_height), interpolation=cv2.INTER_LINEAR)
def _decode_images_if_needed(images, image_height, image_width):
frames = np.asarray(decode_image_bit(images))
if frames.ndim == 3:
frames = frames[None, ...]
if frames.ndim != 4:
raise ValueError(f"Expected decoded frames with shape [T,H,W,3], got {frames.shape}")
if frames.dtype != np.uint8:
frames = frames.astype(np.uint8)
if frames.shape[1:3] != (image_height, image_width):
frames = np.stack(
[_resize_image(frame, image_height, image_width) for frame in frames],
axis=0,
)
return frames
def _empty_image_sequence(num_frames, image_height, image_width):
return np.zeros((num_frames, image_height, image_width, 3), dtype=np.uint8)
def _probe_raw_image_hw(target_inputs, data_type, data_version):
"""Return (H, W) from the first readable camera frame, without resizing."""
for _, _, _, _, input_files in target_inputs:
for input_path in input_files:
data = load(str(input_path), data_type=data_type, data_version=data_version)
for camera_name in CAMERA_CANDIDATES:
for keys in CAMERA_CANDIDATES[camera_name]:
value = _get_nested(data, *keys)
if value is None:
continue
frames = np.asarray(decode_image_bit(value))
if frames.ndim == 3:
return int(frames.shape[0]), int(frames.shape[1])
if frames.ndim == 4 and frames.shape[0] > 0:
return int(frames.shape[1]), int(frames.shape[2])
return DEFAULT_IMAGE_HEIGHT, DEFAULT_IMAGE_WIDTH
def _parse_resolution(text: str) -> tuple[int, int]:
"""Parse '240x320' / '240,320' into (height, width)."""
normalized = text.lower().replace(",", "x").replace("*", "x")
parts = [p.strip() for p in normalized.split("x") if p.strip()]
if len(parts) != 2:
raise argparse.ArgumentTypeError(
f"resolution must look like HxW (e.g. 240x320), got: {text}"
)
height, width = int(parts[0]), int(parts[1])
if height <= 0 or width <= 0:
raise argparse.ArgumentTypeError(f"invalid resolution: {text}")
return height, width
def _resolve_image_hw(args, target_inputs) -> tuple[int, int]:
if args.resolution is not None:
image_hw = _parse_resolution(args.resolution)
elif args.image_height is not None or args.image_width is not None:
if args.image_height is None or args.image_width is None:
raise ValueError("Pass both --image_height and --image_width, or use --resolution")
image_hw = (args.image_height, args.image_width)
else:
image_hw = _probe_raw_image_hw(
target_inputs,
args.data_type,
args.data_version,
)
if image_hw not in SUPPORTED_IMAGE_SIZES:
supported = ", ".join(f"{h}x{w}" for h, w in SUPPORTED_IMAGE_SIZES)
print(
f"[warn] Using image size {image_hw[0]}x{image_hw[1]}; "
f"commonly tested sizes are: {supported}"
)
return image_hw
def _concat_state_parts(parts, name):
valid_parts = []
horizon = None
for part_name, value in parts:
if value is None:
continue
arr = _ensure_2d_float32(value, f"{name}.{part_name}")
if horizon is None:
horizon = arr.shape[0]
elif arr.shape[0] != horizon:
raise ValueError(
f"{name}.{part_name} horizon mismatch: expected {horizon}, got {arr.shape[0]}"
)
valid_parts.append(arr)
if not valid_parts:
return None
return np.concatenate(valid_parts, axis=1)
def _extract_qpos(data):
state_qpos = _concat_state_parts(
[
("left_arm_joint_states", _get_nested(data, "state", "left_arm_joint_states")),
("left_ee_joint_states", _get_nested(data, "state", "left_ee_joint_states")),
("right_arm_joint_states", _get_nested(data, "state", "right_arm_joint_states")),
("right_ee_joint_states", _get_nested(data, "state", "right_ee_joint_states")),
],
"state",
)
if state_qpos is not None:
return state_qpos
raise ValueError("Cannot find qpos data")
def _extract_action(data):
action = _concat_state_parts(
[
("left_arm_joint_states", _get_nested(data, "action", "left_arm_joint_states")),
("left_ee_joint_states", _get_nested(data, "action", "left_ee_joint_states")),
("right_arm_joint_states", _get_nested(data, "action", "right_arm_joint_states")),
("right_ee_joint_states", _get_nested(data, "action", "right_ee_joint_states")),
],
"action",
)
if action is not None:
return action
raise ValueError("Cannot find action data")
def add_frame_compat(dataset: LeRobotDataset, frame: dict, task: str) -> None:
try:
dataset.add_frame(frame, task=task)
except TypeError:
frame = {**frame, "task": task}
dataset.add_frame(frame)
def save_episode_compat(dataset: LeRobotDataset, task: str) -> None:
try:
dataset.save_episode(task=task)
except TypeError:
dataset.save_episode()
def release_dataset_memory(dataset: LeRobotDataset) -> None:
# Some LeRobot variants keep appending every saved episode to `hf_dataset`,
# which makes RAM usage grow linearly until the process gets OOM-killed.
if hasattr(dataset, "create_hf_dataset") and hasattr(dataset, "hf_dataset"):
try:
dataset.hf_dataset = dataset.create_hf_dataset()
except Exception:
pass
def convert_one(
input_path,
dataset,
data_type,
data_version,
current_dims,
target_dims,
image_height,
image_width,
):
data = load(str(input_path), data_type=data_type, data_version=data_version)
state = _extract_qpos(data)
action = _extract_action(data)
if state.shape != action.shape:
raise ValueError(f"state/action shape mismatch: {state.shape} vs {action.shape}")
state = _pad_state_to_target_dims(state, current_dims, target_dims, "state")
action = _pad_state_to_target_dims(action, current_dims, target_dims, "action")
instruction = _choose_instruction(data)
if not instruction:
raise ValueError("No instruction found in data")
images = {}
for camera_name in CAMERA_CANDIDATES:
image_array = _find_camera_array(data, camera_name, image_height, image_width)
if image_array is not None:
images[camera_name] = image_array
num_frames = state.shape[0]
for camera_name in CAMERA_CANDIDATES:
if camera_name not in images:
images[camera_name] = _empty_image_sequence(num_frames, image_height, image_width)
for i in range(num_frames):
frame = {
"observation.state": np.asarray(state[i]),
"action": np.asarray(action[i]),
}
for image_name, image_array in images.items():
frame[f"observation.images.{image_name}"] = image_array[i]
add_frame_compat(dataset, frame, instruction)
save_episode_compat(dataset, instruction)
release_dataset_memory(dataset)
def find_input_files(input_dir):
input_dir = Path(input_dir)
files = sorted(input_dir.rglob("*.hdf5"))
files.extend(sorted(input_dir.rglob("*.h5")))
unique_files = []
seen = set()
for file_path in files:
resolved = str(file_path.resolve())
if resolved not in seen:
seen.add(resolved)
unique_files.append(file_path)
return unique_files
def _collect_target_input_files(targets):
collected = []
for bench_name, task_name, env_cfg_type in targets:
input_dir = _resolve_input_dir(bench_name, task_name, env_cfg_type)
input_files = find_input_files(input_dir)
collected.append((bench_name, task_name, env_cfg_type, input_dir, input_files))
return collected
def _print_matched_targets(target_inputs):
print("Matched targets:")
for bench_name, task_name, env_cfg_type, input_dir, input_files in target_inputs:
print(f" - {bench_name}/{task_name}/{env_cfg_type}: {len(input_files)} files from {input_dir}")
def main():
parser = argparse.ArgumentParser(description="Transform RoboDojo dataset to LeRobot v2.1 format.")
parser.add_argument(
"patterns",
nargs="+",
help='Match expressions like "RoboDojo.*.arx_x5" or "RoboDojo.*.*"',
)
parser.add_argument("--repo_id", type=str, default=None, help="LeRobot repo_id. Defaults to unified_<dataset> patterns summary")
parser.add_argument("--data_type", type=str, default=DEFAULT_DATASET_NAME, help="Dataset type, e.g. RoboDojo")
parser.add_argument("--data_version", type=str, default="v1.0", help="Dataset version, e.g. v1.0")
parser.add_argument("--max_episode", type=int, default=200, help="Max episodes per task/env")
parser.add_argument(
"--resolution",
type=str,
default=None,
help="Target image size as HxW, e.g. 240x320 or 480x640. "
"Default: auto-detect from the first source frame.",
)
parser.add_argument(
"--image_height",
type=int,
default=None,
help="Override target image height (use with --image_width).",
)
parser.add_argument(
"--image_width",
type=int,
default=None,
help="Override target image width (use with --image_height).",
)
args = parser.parse_args()
targets = _discover_conversion_targets(args.patterns)
if not targets:
raise FileNotFoundError(f"No matching dataset/task/env_cfg targets found for patterns: {args.patterns}")
metadata_by_target, target_dims, max_fps = _plan_target_metadata(targets)
target_inputs = _collect_target_input_files(targets)
_print_matched_targets(target_inputs)
image_height, image_width = _resolve_image_hw(args, target_inputs)
print(f"Image size: {image_height}x{image_width}")
repo_id = args.repo_id or f"unified_{'_'.join(pattern.replace('*', 'all').replace('.', '_') for pattern in args.patterns)}".lower()
motors = _build_motor_names_from_dims(target_dims)
dataset = create_empty_dataset(
repo_id,
robot_type="unified_robot",
motors=motors,
fps=max_fps or 50,
mode="video",
image_height=image_height,
image_width=image_width,
dataset_config=DEFAULT_DATASET_CONFIG,
)
failures = []
total_files = 0
total_success = 0
for bench_name, task_name, env_cfg_type, input_dir, input_files in target_inputs:
task_success = 0
if not input_files:
failures.append((str(input_dir), "No .hdf5 or .h5 files found"))
continue
current_dims = metadata_by_target[(bench_name, task_name, env_cfg_type)]["per_arm_dims"]
for input_path in tqdm(input_files, desc=f"Converting {bench_name}/{task_name}/{env_cfg_type}"):
total_files += 1
if task_success >= args.max_episode:
print(
f"Reached max_episode={args.max_episode} "
f"for {bench_name}/{task_name}/{env_cfg_type}"
)
break
try:
convert_one(
input_path,
dataset,
args.data_type,
args.data_version,
current_dims,
target_dims,
image_height,
image_width,
)
task_success += 1
total_success += 1
except Exception as exc:
print(exc)
failures.append((str(input_path), str(exc)))
print(f"Unified target dims per arm: {target_dims}")
print(f"Converted {total_success}/{total_files} files to {HF_LEROBOT_HOME / repo_id}")
if failures:
print("Failed files:")
for file_path, reason in failures:
print(f" - {file_path}: {reason}")
if __name__ == "__main__":
main()

View File

@ -0,0 +1,999 @@
import argparse
import dataclasses
import fnmatch
import random
import shutil
import sys
from pathlib import Path
from typing import Any, Literal
import cv2
import numpy as np
from tqdm import tqdm
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from lerobot.utils.constants import HF_LEROBOT_HOME
from lerobot.datasets.lerobot_dataset import LeRobotDataset
from XPolicyLab.utils.data_loader import load
from XPolicyLab.utils.load_file import load_json, load_yaml
from XPolicyLab.utils.process_data import decode_image_bit
DEFAULT_DATASET_NAME = "RoboDojo"
DATA_ROOT = PROJECT_ROOT / "data"
ENV_CFG_ROOT = PROJECT_ROOT / "env_cfg"
ROBOT_INFO_PATH = ENV_CFG_ROOT / "robot" / "_robot_info.json"
DEFAULT_IMAGE_WIDTH = 640
DEFAULT_IMAGE_HEIGHT = 480
# Common source resolutions: 480x640 (RoboDojo) and 240x320 (RoboTwin).
SUPPORTED_IMAGE_SIZES = ((480, 640), (240, 320))
CAMERA_CANDIDATES = {
"cam_high": [
("vision", "cam_head", "colors"),
],
"cam_left_wrist": [
("vision", "cam_left_wrist", "colors"),
],
"cam_right_wrist": [
("vision", "cam_right_wrist", "colors"),
],
}
# ============================================================
# Fast community-recommended v3.0 config
# ============================================================
@dataclasses.dataclass(frozen=True)
class DatasetConfig:
use_videos: bool = True
tolerance_s: float = 0.0001
# IMPORTANT:
# 0 = disable multiprocessing
image_writer_processes: int = 8
# Usually 2 is enough
image_writer_threads: int = 2
# Use streaming encoding so CRF can be overridden from this script
# instead of relying on LeRobot's internal default of 30.
streaming_encoding: bool = True
video_crf: int | None = 18
# pyav is the fastest & most stable in community
video_backend: str | None = "h264_nvenc"
DEFAULT_DATASET_CONFIG = DatasetConfig()
# ============================================================
# Metadata
# ============================================================
def _load_env_metadata(env_cfg_type):
env_cfg = load_yaml(str(ENV_CFG_ROOT / f"{env_cfg_type}.yml"))
robot_name = env_cfg["config"]["robot"]
robot_action_dim_info = load_json(str(ROBOT_INFO_PATH))[robot_name]
fps = env_cfg.get("observation", {}).get("collect_freq", 50)
return robot_name, robot_action_dim_info, fps
def _split_pattern(pattern):
parts = pattern.split(".")
if len(parts) != 3:
raise ValueError(
f"Pattern should be <dataset>.<task>.<env_cfg>, got: {pattern}"
)
return parts[0], parts[1], parts[2]
def _match_name(name, pattern):
return pattern == "*" or fnmatch.fnmatchcase(name, pattern)
def _discover_conversion_targets(patterns):
targets = []
seen = set()
for pattern in patterns:
dataset_pattern, task_pattern, env_pattern = _split_pattern(pattern)
if not DATA_ROOT.exists():
break
for dataset_dir in sorted(path for path in DATA_ROOT.iterdir() if path.is_dir()):
if not _match_name(dataset_dir.name, dataset_pattern):
continue
for task_dir in sorted(path for path in dataset_dir.iterdir() if path.is_dir()):
if not _match_name(task_dir.name, task_pattern):
continue
for env_dir in sorted(path for path in task_dir.iterdir() if path.is_dir()):
if not _match_name(env_dir.name, env_pattern):
continue
input_dir = env_dir / "data"
if not input_dir.is_dir():
continue
target = (
dataset_dir.name,
task_dir.name,
env_dir.name,
)
if target not in seen:
seen.add(target)
targets.append(target)
return targets
def _dims_from_robot_action_info(robot_action_dim_info):
arm_dims = robot_action_dim_info.get("arm_dim", [])
ee_dims = robot_action_dim_info.get("ee_dim", [])
per_arm_dims = []
for index, arm_dim in enumerate(arm_dims):
ee_dim = ee_dims[index] if index < len(ee_dims) else 0
per_arm_dims.append(arm_dim + ee_dim)
return per_arm_dims
def _plan_target_metadata(targets):
max_per_arm_dims = []
metadata = {}
max_fps = 0
for bench_name, task_name, env_cfg_type in targets:
robot_name, robot_action_dim_info, fps = _load_env_metadata(
env_cfg_type
)
per_arm_dims = _dims_from_robot_action_info(
robot_action_dim_info
)
if len(max_per_arm_dims) < len(per_arm_dims):
max_per_arm_dims.extend(
[0] * (len(per_arm_dims) - len(max_per_arm_dims))
)
for index, dim in enumerate(per_arm_dims):
max_per_arm_dims[index] = max(
max_per_arm_dims[index],
dim,
)
max_fps = max(max_fps, fps)
metadata[(bench_name, task_name, env_cfg_type)] = {
"robot_name": robot_name,
"robot_action_dim_info": robot_action_dim_info,
"fps": fps,
"per_arm_dims": per_arm_dims,
}
return metadata, max_per_arm_dims, max_fps
# ============================================================
# Motors
# ============================================================
def _build_motor_names_from_dims(per_arm_dims):
if not per_arm_dims:
raise ValueError("per_arm_dims is empty")
if len(per_arm_dims) == 1:
prefixes = ["arm"]
elif len(per_arm_dims) == 2:
prefixes = ["left", "right"]
else:
prefixes = [f"arm_{index}" for index in range(len(per_arm_dims))]
motors = []
for index, total_dim in enumerate(per_arm_dims):
prefix = prefixes[index]
for joint_idx in range(total_dim):
motors.append(f"{prefix}_joint_{joint_idx}")
return motors
# ============================================================
# Utils
# ============================================================
def _resolve_input_dir(bench_name, task_name, env_cfg_type, input_dir=None):
if input_dir is not None:
return Path(input_dir)
return DATA_ROOT / bench_name / task_name / env_cfg_type / "data"
def _get_nested(data, *keys, default=None):
cur = data
for key in keys:
if not isinstance(cur, dict) or key not in cur:
return default
cur = cur[key]
return cur
def _ensure_2d_float32(array, name):
arr = np.asarray(array, dtype=np.float32)
if arr.ndim == 1:
arr = arr[:, None]
if arr.ndim != 2:
raise ValueError(f"{name} should be 2D, got shape {arr.shape}")
return arr
def _ensure_utf8_strings(value):
if value is None:
return []
if isinstance(value, str):
return [value]
if isinstance(value, bytes):
return [value.decode("utf-8")]
if isinstance(value, np.ndarray):
if value.ndim == 0:
return _ensure_utf8_strings(value.item())
return [
str(
x.decode("utf-8")
if isinstance(x, (bytes, np.bytes_))
else x
)
for x in value.tolist()
]
if isinstance(value, (list, tuple)):
result = []
for item in value:
result.extend(_ensure_utf8_strings(item))
return result
return [str(value)]
def _find_instructions(data):
candidates = [
_get_nested(data, "instruction"),
_get_nested(data, "instructions"),
]
for candidate in candidates:
strings = [
s for s in _ensure_utf8_strings(candidate)
if s
]
if strings:
return strings
return []
def _choose_instruction(data):
instructions = _find_instructions(data)
if not instructions:
return ""
return random.choice(instructions)
# ============================================================
# State / Action
# ============================================================
def _concat_state_parts(parts, name):
valid_parts = []
horizon = None
for part_name, value in parts:
if value is None:
continue
arr = _ensure_2d_float32(
value,
f"{name}.{part_name}",
)
if horizon is None:
horizon = arr.shape[0]
elif arr.shape[0] != horizon:
raise ValueError(
f"{name}.{part_name} horizon mismatch: "
f"expected {horizon}, got {arr.shape[0]}"
)
valid_parts.append(arr)
if not valid_parts:
return None
return np.concatenate(valid_parts, axis=1)
def _extract_qpos(data):
state_qpos = _concat_state_parts(
[
("left_arm_joint_states", _get_nested(data, "state", "left_arm_joint_states")),
("left_ee_joint_states", _get_nested(data, "state", "left_ee_joint_states")),
("right_arm_joint_states", _get_nested(data, "state", "right_arm_joint_states")),
("right_ee_joint_states", _get_nested(data, "state", "right_ee_joint_states")),
],
"state",
)
if state_qpos is not None:
return state_qpos
raise ValueError("Cannot find qpos data")
def _extract_action(data):
action = _concat_state_parts(
[
("left_arm_joint_states", _get_nested(data, "action", "left_arm_joint_states")),
("left_ee_joint_states", _get_nested(data, "action", "left_ee_joint_states")),
("right_arm_joint_states", _get_nested(data, "action", "right_arm_joint_states")),
("right_ee_joint_states", _get_nested(data, "action", "right_ee_joint_states")),
],
"action",
)
if action is not None:
return action
raise ValueError("Cannot find action data")
def _pad_state_to_target_dims(array, current_dims, target_dims, name):
arr = _ensure_2d_float32(array, name)
expected_dim = sum(current_dims)
if arr.shape[1] != expected_dim:
raise ValueError(
f"{name} dim mismatch: "
f"expected {expected_dim}, got {arr.shape[1]}"
)
padded_parts = []
start = 0
for arm_index, target_dim in enumerate(target_dims):
current_dim = (
current_dims[arm_index]
if arm_index < len(current_dims)
else 0
)
end = start + current_dim
part = (
arr[:, start:end]
if current_dim
else np.zeros((arr.shape[0], 0), dtype=np.float32)
)
start = end
if target_dim > current_dim:
padding = np.zeros(
(arr.shape[0], target_dim - current_dim),
dtype=np.float32,
)
part = np.concatenate([part, padding], axis=1)
padded_parts.append(part)
return np.concatenate(padded_parts, axis=1)
# ============================================================
# Images
# ============================================================
def _resize_image(image, image_height, image_width):
if image.shape[:2] == (image_height, image_width):
return image
return cv2.resize(
image,
(image_width, image_height),
interpolation=cv2.INTER_LINEAR,
)
def _decode_images_if_needed(images, image_height, image_width):
frames = np.asarray(decode_image_bit(images))
if frames.ndim == 3:
frames = frames[None, ...]
if frames.ndim != 4:
raise ValueError(f"Expected decoded frames with shape [T,H,W,3], got {frames.shape}")
if frames.dtype != np.uint8:
frames = frames.astype(np.uint8)
if frames.shape[1:3] != (image_height, image_width):
frames = np.stack(
[_resize_image(frame, image_height, image_width) for frame in frames],
axis=0,
)
return frames
def _find_camera_array(data, camera_name, image_height, image_width):
for keys in CAMERA_CANDIDATES[camera_name]:
value = _get_nested(data, *keys)
if value is not None:
return _decode_images_if_needed(value, image_height, image_width)
return None
def _probe_raw_image_hw(target_inputs, data_type, data_version):
"""Return (H, W) from the first readable camera frame, without resizing."""
for _, _, _, _, input_files in target_inputs:
for input_path in input_files:
data = load(
str(input_path),
data_type=data_type,
data_version=data_version,
)
for camera_name in CAMERA_CANDIDATES:
for keys in CAMERA_CANDIDATES[camera_name]:
value = _get_nested(data, *keys)
if value is None:
continue
frames = np.asarray(decode_image_bit(value))
if frames.ndim == 3:
return int(frames.shape[0]), int(frames.shape[1])
if frames.ndim == 4 and frames.shape[0] > 0:
return int(frames.shape[1]), int(frames.shape[2])
return DEFAULT_IMAGE_HEIGHT, DEFAULT_IMAGE_WIDTH
def _parse_resolution(text: str) -> tuple[int, int]:
"""Parse '240x320' / '240,320' into (height, width)."""
normalized = text.lower().replace(",", "x").replace("*", "x")
parts = [p.strip() for p in normalized.split("x") if p.strip()]
if len(parts) != 2:
raise argparse.ArgumentTypeError(
f"resolution must look like HxW (e.g. 240x320), got: {text}"
)
height, width = int(parts[0]), int(parts[1])
if height <= 0 or width <= 0:
raise argparse.ArgumentTypeError(f"invalid resolution: {text}")
return height, width
def _resolve_image_hw(args, target_inputs) -> tuple[int, int]:
if args.resolution is not None:
image_hw = _parse_resolution(args.resolution)
elif args.image_height is not None or args.image_width is not None:
if args.image_height is None or args.image_width is None:
raise ValueError("Pass both --image_height and --image_width, or use --resolution")
image_hw = (args.image_height, args.image_width)
else:
image_hw = _probe_raw_image_hw(
target_inputs,
args.data_type,
args.data_version,
)
if image_hw not in SUPPORTED_IMAGE_SIZES:
supported = ", ".join(f"{h}x{w}" for h, w in SUPPORTED_IMAGE_SIZES)
print(
f"[warn] Using image size {image_hw[0]}x{image_hw[1]}; "
f"commonly tested sizes are: {supported}"
)
return image_hw
# ============================================================
# Dataset
# ============================================================
def create_empty_dataset(
repo_id: str,
robot_type: str,
motors: list[str],
fps: int,
mode: Literal["video", "image"] = "video",
*,
image_height: int = DEFAULT_IMAGE_HEIGHT,
image_width: int = DEFAULT_IMAGE_WIDTH,
dataset_config: DatasetConfig = DEFAULT_DATASET_CONFIG,
) -> Any:
features = {
"observation.state": {
"dtype": "float32",
"shape": (len(motors),),
"names": [motors],
},
"action": {
"dtype": "float32",
"shape": (len(motors),),
"names": [motors],
},
}
for cam in CAMERA_CANDIDATES.keys():
features[f"observation.images.{cam}"] = {
"dtype": mode,
"shape": (
3,
image_height,
image_width,
),
"names": [
"channels",
"height",
"width",
],
}
dataset_root = Path(HF_LEROBOT_HOME) / repo_id
if dataset_root.exists():
shutil.rmtree(dataset_root)
return LeRobotDataset.create(
repo_id=repo_id,
fps=fps,
robot_type=robot_type,
features=features,
use_videos=dataset_config.use_videos,
tolerance_s=dataset_config.tolerance_s,
image_writer_processes=dataset_config.image_writer_processes,
image_writer_threads=dataset_config.image_writer_threads,
streaming_encoding=dataset_config.streaming_encoding,
video_backend=dataset_config.video_backend,
)
def configure_video_encoding(dataset: Any, dataset_config: DatasetConfig) -> None:
if not dataset_config.use_videos:
return
streaming_encoder = getattr(dataset, "_streaming_encoder", None)
if streaming_encoder is None:
raise RuntimeError(
"Video CRF override requires LeRobot streaming_encoding=True, "
"but no streaming encoder was initialized."
)
streaming_encoder.crf = dataset_config.video_crf
def finalize_dataset(dataset: Any) -> None:
if hasattr(dataset, "stop_image_writer"):
dataset.stop_image_writer()
meta = getattr(dataset, "meta", None)
if meta is not None and hasattr(meta, "_close_writer"):
meta._close_writer()
# ============================================================
# Convert
# ============================================================
def convert_one(
input_path,
dataset,
data_type,
data_version,
current_dims,
target_dims,
image_height,
image_width,
):
data = load(
str(input_path),
data_type=data_type,
data_version=data_version,
)
state = _extract_qpos(data)
action = _extract_action(data)
if state.shape != action.shape:
raise ValueError(
f"state/action mismatch: "
f"{state.shape} vs {action.shape}"
)
state = _pad_state_to_target_dims(
state,
current_dims,
target_dims,
"state",
)
action = _pad_state_to_target_dims(
action,
current_dims,
target_dims,
"action",
)
instruction = _choose_instruction(data)
if not instruction:
raise ValueError("No instruction found")
images = {}
for camera_name in CAMERA_CANDIDATES:
image_array = _find_camera_array(
data,
camera_name,
image_height,
image_width,
)
if image_array is not None:
images[camera_name] = image_array
num_frames = state.shape[0]
for index in range(num_frames):
frame = {
"observation.state": state[index],
"action": action[index],
"task": instruction,
}
for image_name, image_array in images.items():
if index >= len(image_array):
continue
frame[f"observation.images.{image_name}"] = image_array[index]
dataset.add_frame(frame)
dataset.save_episode()
# ============================================================
# Input files
# ============================================================
def find_input_files(input_dir):
input_dir = Path(input_dir)
files = sorted(input_dir.rglob("*.hdf5"))
files.extend(sorted(input_dir.rglob("*.h5")))
unique_files = []
seen = set()
for file_path in files:
resolved = str(file_path.resolve())
if resolved not in seen:
seen.add(resolved)
unique_files.append(file_path)
return unique_files
def _collect_target_input_files(targets):
collected = []
for bench_name, task_name, env_cfg_type in targets:
input_dir = _resolve_input_dir(
bench_name,
task_name,
env_cfg_type,
)
input_files = find_input_files(input_dir)
collected.append(
(
bench_name,
task_name,
env_cfg_type,
input_dir,
input_files,
)
)
return collected
def _print_matched_targets(target_inputs):
print("Matched targets:")
for (
bench_name,
task_name,
env_cfg_type,
input_dir,
input_files,
) in target_inputs:
print(
f" - {bench_name}/{task_name}/{env_cfg_type}: "
f"{len(input_files)} files from {input_dir}"
)
# ============================================================
# Main
# ============================================================
def main():
parser = argparse.ArgumentParser(
description="Transform RoboDojo dataset to LeRobot v3.0 format."
)
parser.add_argument(
"patterns",
nargs="+",
help='Match expressions like "RoboDojo.*.arx_x5"',
)
parser.add_argument(
"--repo_id",
type=str,
default=None,
)
parser.add_argument(
"--data_type",
type=str,
default=DEFAULT_DATASET_NAME,
)
parser.add_argument(
"--data_version",
type=str,
default="v1.0",
)
parser.add_argument(
"--max_episode",
type=int,
default=200,
)
parser.add_argument(
"--resolution",
type=str,
default=None,
help="Target image size as HxW, e.g. 240x320 or 480x640. "
"Default: auto-detect from the first source frame.",
)
parser.add_argument(
"--image_height",
type=int,
default=None,
help="Override target image height (use with --image_width).",
)
parser.add_argument(
"--image_width",
type=int,
default=None,
help="Override target image width (use with --image_height).",
)
args = parser.parse_args()
targets = _discover_conversion_targets(args.patterns)
if not targets:
raise FileNotFoundError(
f"No matching targets found: {args.patterns}"
)
metadata_by_target, target_dims, max_fps = (
_plan_target_metadata(targets)
)
target_inputs = _collect_target_input_files(targets)
_print_matched_targets(target_inputs)
image_height, image_width = _resolve_image_hw(args, target_inputs)
print(f"Image size: {image_height}x{image_width}")
repo_id = (
args.repo_id
or f"unified_{'_'.join(pattern.replace('*', 'all').replace('.', '_') for pattern in args.patterns)}".lower()
)
motors = _build_motor_names_from_dims(target_dims)
dataset = create_empty_dataset(
repo_id=repo_id,
robot_type="unified_robot",
motors=motors,
fps=max_fps or 50,
# IMPORTANT
mode="video",
image_height=image_height,
image_width=image_width,
dataset_config=DEFAULT_DATASET_CONFIG,
)
configure_video_encoding(dataset, DEFAULT_DATASET_CONFIG)
failures = []
total_success = 0
total_files = 0
try:
for (
bench_name,
task_name,
env_cfg_type,
input_dir,
input_files,
) in target_inputs:
task_success = 0
if not input_files:
failures.append(
(
str(input_dir),
"No .hdf5 or .h5 files found",
)
)
continue
current_dims = metadata_by_target[
(
bench_name,
task_name,
env_cfg_type,
)
]["per_arm_dims"]
for input_path in tqdm(
input_files,
desc=f"Converting {bench_name}/{task_name}/{env_cfg_type}",
):
total_files += 1
if task_success >= args.max_episode:
print(
"Reached "
f"max_episode={args.max_episode} "
f"for {bench_name}/{task_name}/{env_cfg_type}"
)
break
try:
convert_one(
input_path,
dataset,
args.data_type,
args.data_version,
current_dims,
target_dims,
image_height,
image_width,
)
task_success += 1
total_success += 1
except Exception as exc:
print(exc)
failures.append(
(
str(input_path),
str(exc),
)
)
finally:
finalize_dataset(dataset)
print(f"Unified target dims per arm: {target_dims}")
print(
f"Converted {total_success}/{total_files} "
f"files to {Path(HF_LEROBOT_HOME) / repo_id}"
)
if failures:
print("Failed files:")
for file_path, reason in failures:
print(f" - {file_path}: {reason}")
if __name__ == "__main__":
main()

View File

@ -0,0 +1,168 @@
import asyncio
import os
import threading
import ast
import time
import yaml
import importlib
import argparse
import traceback
from client_server.tcp.model_server import ModelServer
def _default_protocol() -> str:
"""Default to the websocket policy protocol."""
return "ws"
def eval_function_decorator(policy_model_name, Func_and_Class_name):
"""Load a specified function (e.g., get_model) from a policy module"""
module = importlib.import_module(policy_model_name)
return getattr(module, Func_and_Class_name)
def main(deploy_cfg):
"""Main entry: load model, start server, run indefinitely"""
# Extract basic arguments
policy_name = deploy_cfg.get("policy_name")
port = deploy_cfg.get("port")
host = deploy_cfg.get("host", "0.0.0.0")
protocol = deploy_cfg.get("protocol", "ws")
# Instantiate model
model_class_func = eval_function_decorator(f"XPolicyLab.policy.{policy_name}.model", "Model")
model = model_class_func(deploy_cfg)
if protocol == "ws":
try:
from client_server.ws.model_server import PolicyServer, PolicyServerConfig
except ModuleNotFoundError as exc:
if exc.name == "client_server":
# client_server.ws ships in this repo; make it importable even when
# XPolicyLab is not pip-installed in the current environment.
import sys
repo_root = os.path.dirname(os.path.abspath(__file__))
if repo_root not in sys.path:
sys.path.insert(0, repo_root)
try:
from client_server.ws.model_server import PolicyServer, PolicyServerConfig
except ModuleNotFoundError as dep_exc:
raise RuntimeError(
"ws policy server requires XPolicyLab websocket dependencies "
f"(missing module: {dep_exc.name}). Install in the policy env with: "
"pip install -e . from the XPolicyLab root."
) from dep_exc
else:
raise RuntimeError(
"ws policy server requires XPolicyLab websocket dependencies "
f"(missing module: {exc.name}). Install in the policy env with: "
"pip install -e . from the XPolicyLab root."
) from exc
server = PolicyServer(
model,
PolicyServerConfig(
host=host,
port=int(port),
ws_ping_interval_s=deploy_cfg.get("ws_ping_interval_s", 20.0),
ws_ping_timeout_s=deploy_cfg.get("ws_ping_timeout_s", 20.0),
),
)
try:
asyncio.run(server.serve_forever())
except KeyboardInterrupt:
print("\nShutting down websocket policy server...")
return
if protocol != "legacy_tcp":
raise ValueError(f"unsupported policy server protocol: {protocol}")
# Wrap server.start so exceptions inside thread are fully printed
def run_server():
try:
server.start()
except Exception:
print("\033[31m[ERROR] Exception occurred inside server thread:\033[0m")
traceback.print_exc()
raise
# Start server in background thread
server = ModelServer(model, host=host, port=port)
thread = threading.Thread(target=run_server, daemon=True)
thread.start()
# Keep main thread alive until KeyboardInterrupt
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print("\n🛑 Shutting down server...")
server.stop()
thread.join()
def parse_args_and_config():
"""Parse CLI args and YAML config, merge overrides"""
parser = argparse.ArgumentParser()
parser.add_argument("--config_path", "--config-path", dest="config_path", type=str, required=True, help="Path to config YAML")
parser.add_argument("--protocol", choices=("legacy_tcp", "ws"), help="Policy server protocol")
parser.add_argument("--host", help="Policy server bind host")
parser.add_argument("--port", type=int, help="Policy server bind port")
parser.add_argument("--relay-url", dest="relay_url", help="Relay URL for future relay mode")
parser.add_argument("--overrides", nargs=argparse.REMAINDER, help="Override config values")
args = parser.parse_args()
# Load base config
with open(args.config_path, "r", encoding="utf-8") as f:
cfg = yaml.safe_load(f)
# Parse overrides: --key value pairs
def _parse_val(s: str):
# safer than eval; supports numbers/bool/None/list/dict when properly quoted
try:
return ast.literal_eval(s)
except Exception:
return s
if args.overrides:
tokens = args.overrides
# Case A: key=value key=value ...
if all(("=" in t and not t.startswith("-")) for t in tokens):
for t in tokens:
k, v = t.split("=", 1)
cfg[k] = _parse_val(v)
else:
# Case B: --key value --key value ...
if len(tokens) % 2 != 0:
raise ValueError(f"--overrides expects key value pairs, got: {tokens}")
it = iter(tokens)
for key in it:
val = next(it)
cfg[key.lstrip("-")] = _parse_val(val)
if args.protocol is not None:
cfg["protocol"] = args.protocol
else:
cfg.setdefault("protocol", _default_protocol())
if args.host is not None:
cfg["host"] = args.host
if args.port is not None:
cfg["port"] = args.port
if args.relay_url is not None:
cfg["relay_url"] = args.relay_url
def _require_non_empty(key: str):
if key not in cfg:
raise ValueError(f"{key} must be specified in config or overrides")
val = cfg[key]
if val is None:
raise ValueError(f"{key} must be non-empty")
if isinstance(val, str) and not val.strip():
raise ValueError(f"{key} must be non-empty")
_require_non_empty("host")
_require_non_empty("port")
return cfg
if __name__ == "__main__":
deploy_cfg = parse_args_and_config()
main(deploy_cfg)

View File

@ -0,0 +1,183 @@
"""Unified checkpoint-directory resolution shared by all policy adapters.
Every adapter resolves the checkpoint (run) directory from the deploy config in
the same way. Resolution precedence (highest first):
1. An explicit path key in the deploy config
(``model_path`` / ``checkpoint_path`` / ``ckpt_path`` / ``model_dir`` /
``pretrained_path``).
2. ``ckpt_name`` given as a *path* (absolute, or a relative path containing a
path separator). Relative paths resolve against the policy directory.
3. The concatenated run-dir name built from the eval args:
``{bench_name}-{ckpt_name}-{env_cfg_type}-{action_type}-{seed}``.
4. ``checkpoints/<ckpt_name>`` (the verbatim full run-dir name) as a fallback,
so checkpoints saved/named directly by their run id still load.
Training scripts write to the same concatenated name
(``{bench_name}-{ckpt_name}-{env_cfg_type}-{action_type}-{seed}``), so eval finds
the checkpoint without the caller having to know the exact template.
Adapters with an extra naming layer (e.g. an ``aloha_<name>`` prefix, timestamp
subdirs, or DeepSpeed nesting) should call :func:`build_run_dir_name` for the
shared base name and/or iterate :func:`candidate_checkpoint_roots` and apply
their per-root logic on top.
"""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any, Optional
# Deploy-config keys that carry an explicit checkpoint path (highest priority).
DEFAULT_EXPLICIT_KEYS = (
"model_path",
"checkpoint_path",
"ckpt_path",
"model_dir",
"pretrained_path",
)
def _clean(value: Any) -> Optional[str]:
"""Return a stripped non-empty string, or None."""
if value is None:
return None
text = str(value).strip()
return text or None
def ckpt_name_is_path(ckpt_name: Any) -> bool:
"""True when ``ckpt_name`` should be treated as a filesystem path.
A value counts as a path when it is absolute or contains a path separator
(``/`` or the OS-specific ``os.sep``). A bare identifier such as
``RoboDojo-cotrain-arx_x5-joint-0`` is not a path.
"""
text = _clean(ckpt_name)
if text is None:
return False
expanded = os.path.expanduser(text)
if Path(expanded).is_absolute():
return True
if "/" in text or os.sep in text:
return True
return bool(os.altsep and os.altsep in text)
def build_run_dir_name(model_cfg: dict, *, include_seed: bool = True) -> Optional[str]:
"""Build ``{bench_name}-{ckpt_name}-{env_cfg_type}-{action_type}[-{seed}]``.
Returns None when any required component is missing. ``bench_name`` falls
back to the legacy ``dataset_name`` key for backward compatibility.
"""
bench = _clean(model_cfg.get("bench_name") or model_cfg.get("dataset_name"))
ckpt = _clean(model_cfg.get("ckpt_name"))
env_cfg_type = _clean(model_cfg.get("env_cfg_type"))
action_type = _clean(model_cfg.get("action_type"))
parts = [bench, ckpt, env_cfg_type, action_type]
if include_seed:
parts.append(_clean(model_cfg.get("seed")))
if any(part is None for part in parts):
return None
return "-".join(parts)
def _as_path(value: Any, base_dir: Optional[Path]) -> Path:
path = Path(os.path.expanduser(str(value)))
if not path.is_absolute() and base_dir is not None:
path = Path(base_dir) / path
return path
def candidate_checkpoint_roots(
model_cfg: dict,
checkpoints_dir: os.PathLike | str,
*,
policy_dir: os.PathLike | str | None = None,
explicit_keys: tuple[str, ...] = DEFAULT_EXPLICIT_KEYS,
include_seed: bool = True,
resolve: bool = True,
) -> list[Path]:
"""Return ordered, de-duplicated candidate checkpoint-root paths.
See the module docstring for the precedence. ``policy_dir`` is the base for
resolving relative path inputs (defaults to ``checkpoints_dir``'s parent).
"""
checkpoints_dir = Path(checkpoints_dir)
base_dir = Path(policy_dir) if policy_dir is not None else checkpoints_dir.parent
roots: list[Path] = []
seen: set[str] = set()
def add(path: os.PathLike | str) -> None:
candidate = Path(path)
if resolve:
try:
candidate = candidate.resolve()
except OSError:
pass
key = str(candidate)
if key not in seen:
seen.add(key)
roots.append(candidate)
# 1. explicit path keys
for key in explicit_keys:
value = _clean(model_cfg.get(key))
if value:
add(_as_path(value, base_dir))
ckpt_name = _clean(model_cfg.get("ckpt_name"))
is_path = ckpt_name is not None and ckpt_name_is_path(ckpt_name)
# 2. ckpt_name given as a path (short-circuits the name-based candidates)
if is_path:
add(_as_path(ckpt_name, base_dir))
else:
# 3. concatenated 5-tuple run-dir name
run_dir_name = build_run_dir_name(model_cfg, include_seed=include_seed)
if run_dir_name:
add(checkpoints_dir / run_dir_name)
# 4. fallback: checkpoints/<ckpt_name> (verbatim full run-dir name)
if ckpt_name:
add(checkpoints_dir / ckpt_name)
return roots
def resolve_checkpoint_root(
model_cfg: dict,
checkpoints_dir: os.PathLike | str,
*,
policy_dir: os.PathLike | str | None = None,
explicit_keys: tuple[str, ...] = DEFAULT_EXPLICIT_KEYS,
include_seed: bool = True,
must_exist: bool = True,
) -> Path:
"""Resolve the checkpoint root directory (or file) per the shared precedence.
Returns the first existing candidate. When ``must_exist`` is False and no
candidate exists, returns the highest-priority candidate so callers can raise
their own error. Raises ``FileNotFoundError`` (listing what was checked) when
``must_exist`` is True and nothing resolves.
"""
candidates = candidate_checkpoint_roots(
model_cfg,
checkpoints_dir,
policy_dir=policy_dir,
explicit_keys=explicit_keys,
include_seed=include_seed,
)
for candidate in candidates:
if candidate.exists():
return candidate
if not must_exist and candidates:
return candidates[0]
checked = "\n ".join(str(path) for path in candidates) or " <no checkpoint candidates>"
raise FileNotFoundError(
"Could not resolve a checkpoint directory. Pass a valid ckpt_name "
"(full run-dir name or a path), or set an explicit checkpoint path in "
f"deploy.yml ({'/'.join(explicit_keys)}).\nChecked:\n {checked}"
)

View File

@ -0,0 +1,58 @@
# Load data for data conversion
import h5py, json, argparse
import numpy as np
from XPolicyLab.utils.process_data import decode_image_bit
def load_xspark_v1(hdf5_path, decode_images=True):
def h5_to_dict(obj):
d = {}
for key, item in obj.items():
if isinstance(item, h5py.Group):
d[key] = h5_to_dict(item)
elif isinstance(item, h5py.Dataset):
val = item[()]
if key == "colors" and isinstance(val, np.ndarray):
d[key] = decode_image_bit(val) if decode_images else val
continue
if isinstance(val, (bytes, np.bytes_)):
try:
decoded_str = val.decode("utf-8")
try:
d[key] = json.loads(decoded_str)
except json.JSONDecodeError:
d[key] = decoded_str
except Exception:
d[key] = val
elif isinstance(val, np.ndarray) and val.dtype.kind in ["S", "U", "O"]:
try:
val_item = val.item() if val.size == 1 else val
if isinstance(val_item, (bytes, np.bytes_)):
decoded_str = val_item.decode("utf-8")
elif isinstance(val_item, str):
decoded_str = val_item
else:
d[key] = val
continue
try:
d[key] = json.loads(decoded_str)
except json.JSONDecodeError:
d[key] = decoded_str
except Exception:
d[key] = val
else:
d[key] = val
return d
with h5py.File(hdf5_path, "r") as f:
return h5_to_dict(f)
def load(data_path, data_type="xspark", data_version="v1.0"):
# RoboDojo sim_cloud HDF5 uses the same xspark v1.0 layout.
if data_type in {"xspark", "RoboDojo"}:
if data_version == "v1.0":
return load_xspark_v1(data_path, decode_images=True)
raise NotImplementedError(f"{data_version} is not valid in {data_type} .")
raise NotImplementedError(f"{data_type} is not valid data type. ")

View File

@ -0,0 +1,18 @@
#!/bin/bash
set -e
ROOT_DIR="$1"
env_cfg_type="$2"
python3 -c '
import sys, os, json
root_dir = sys.argv[1]
env_cfg_type = sys.argv[2]
robot_action_dim_info = json.load(
open(os.path.join(root_dir, "XPolicyLab", "utils", "robot", "_robot_info.json"), "r", encoding="utf-8")
)[env_cfg_type]
print(sum(robot_action_dim_info["arm_dim"]) + sum(robot_action_dim_info["ee_dim"]))
' "${ROOT_DIR}" "${env_cfg_type}"

View File

@ -0,0 +1,13 @@
#!/bin/bash
set -e
FREE_PORT=$(python3 - <<'PY'
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("", 0))
print(s.getsockname()[1])
PY
)
echo -e "\033[33m[INFO] Using socket port: ${FREE_PORT}\033[0m" >&2
echo "${FREE_PORT}"

View File

@ -0,0 +1,41 @@
import h5py
import yaml
def load_hdf5(path: str) -> dict:
def _read(obj):
# Dataset -> numpy / scalar / bytes->str
if isinstance(obj, h5py.Dataset):
v = obj[()]
if isinstance(v, (bytes, bytearray)):
return v.decode("utf-8", errors="replace")
# numpy scalar -> python scalar
try:
return v.item()
except Exception:
return v
# Group -> dict
out = {}
for k, v in obj.items():
out[k] = _read(v)
return out
with h5py.File(path, "r") as f:
data = _read(f)
if len(f.attrs) > 0:
data["_attrs"] = {k: f.attrs[k] for k in f.attrs.keys()}
return data
def load_yaml(path: str) -> dict:
with open(path, "r") as f:
data = yaml.safe_load(f)
return data
def load_json(path: str) -> dict:
import json
with open(path, "r") as f:
data = json.load(f)
return data

View File

@ -0,0 +1,461 @@
import numpy as np
import os
import cv2
from XPolicyLab.utils.load_file import load_json, load_yaml
def _validate_config(action_type: str, robot_action_dim_info: dict, source_type: str):
"""
Validate configuration and return normalized values.
Args:
action_type: 'joint' or 'ee'.
robot_action_dim_info: Dict with keys:
- 'arm_dim': list[int]
- 'ee_dim': list[int]
source_type: 'obs' or 'dataset'.
Returns:
arm_dims, ee_dims, num_arms
"""
if action_type not in {"joint", "ee"}:
raise ValueError(
f"Unsupported action_type: {action_type!r}. "
"Supported values are 'joint' and 'ee'."
)
if source_type not in {"obs", "dataset"}:
raise ValueError(
f"Unsupported source_type: {source_type!r}. "
"Supported values are 'obs' and 'dataset'."
)
if "arm_dim" not in robot_action_dim_info or "ee_dim" not in robot_action_dim_info:
raise KeyError("robot_action_dim_info must contain both 'arm_dim' and 'ee_dim'.")
arm_dims = robot_action_dim_info["arm_dim"]
ee_dims = robot_action_dim_info["ee_dim"]
if not isinstance(arm_dims, (list, tuple)) or not isinstance(ee_dims, (list, tuple)):
raise TypeError("'arm_dim' and 'ee_dim' must be list or tuple.")
if len(arm_dims) != len(ee_dims):
raise ValueError(
f"'arm_dim' and 'ee_dim' must have the same length, "
f"got {len(arm_dims)} and {len(ee_dims)}."
)
if len(arm_dims) not in {1, 2}:
raise ValueError(
f"Only single-arm or dual-arm robots are supported, got {len(arm_dims)} arms."
)
if any(d <= 0 for d in arm_dims) or any(d <= 0 for d in ee_dims):
raise ValueError("All dimensions in 'arm_dim' and 'ee_dim' must be positive.")
return list(arm_dims), list(ee_dims), len(arm_dims)
def _get_state_keys(action_type: str, num_arms: int, source_type: str):
"""
Return arm keys and ee keys for the current state schema.
source_type='obs' uses singular keys:
single-arm:
action_type='joint' -> ['joint_state'], ['ee_joint_state']
action_type='ee' -> ['ee_pose'], ['ee_joint_state']
dual-arm:
action_type='joint' -> ['left_arm_joint_state', 'right_arm_joint_state'],
['left_ee_joint_state', 'right_ee_joint_state']
action_type='ee' -> ['left_ee_pose', 'right_ee_pose'],
['left_ee_joint_state', 'right_ee_joint_state']
source_type='dataset' uses plural keys:
single-arm:
action_type='joint' -> ['joint_states'], ['ee_joint_states']
action_type='ee' -> ['ee_poses'], ['ee_joint_states']
dual-arm:
action_type='joint' -> ['left_arm_joint_states', 'right_arm_joint_states'],
['left_ee_joint_states', 'right_ee_joint_states']
action_type='ee' -> ['left_ee_poses', 'right_ee_poses'],
['left_ee_joint_states', 'right_ee_joint_states']
"""
suffix = "" if source_type == "obs" else "s"
if num_arms == 1:
arm_keys = [f"joint_state{suffix}"] if action_type == "joint" else [f"ee_pose{suffix}"]
ee_keys = [f"ee_joint_state{suffix}"]
else:
if action_type == "joint":
arm_keys = [
f"left_arm_joint_state{suffix}",
f"right_arm_joint_state{suffix}",
]
else:
arm_keys = [
f"left_ee_pose{suffix}",
f"right_ee_pose{suffix}",
]
ee_keys = [
f"left_ee_joint_state{suffix}",
f"right_ee_joint_state{suffix}",
]
return arm_keys, ee_keys
def _ensure_valid_state_array(name: str, value, expected_last_dim: int) -> np.ndarray:
"""Convert value to np.ndarray and validate only the last dimension."""
arr = np.asarray(value)
if arr.shape[-1] != expected_last_dim:
raise ValueError(
f"State field '{name}' last dim mismatch: "
f"expected {expected_last_dim}, got {arr.shape[-1]}."
)
return arr
def pack_robot_state(
obs: dict,
action_type: str,
robot_action_dim_info: dict,
source_type: str = "obs",
state_type: str = "state",
) -> np.ndarray:
"""
Pack robot state from obs['state'] into one vector along the last dimension.
Packing order:
single-arm:
[arm_0, ee_0]
dual-arm:
[arm_0, ee_0, arm_1, ee_1]
"""
if state_type not in obs:
raise KeyError(f"Input obs must contain a '{state_type}' field.")
state_dict = obs[state_type]
arm_dims, ee_dims, num_arms = _validate_config(action_type, robot_action_dim_info, source_type)
arm_keys, ee_keys = _get_state_keys(action_type, num_arms, source_type)
parts = []
expected_prefix_shape = None
for i, (arm_key, ee_key, arm_dim, ee_dim) in enumerate(
zip(arm_keys, ee_keys, arm_dims, ee_dims)
):
if arm_key not in state_dict:
raise KeyError(f"Missing key '{arm_key}' in obs['state'] for arm {i}.")
if ee_key not in state_dict:
raise KeyError(f"Missing key '{ee_key}' in obs['state'] for arm {i}.")
arm_value = _ensure_valid_state_array(arm_key, state_dict[arm_key], arm_dim)
ee_value = _ensure_valid_state_array(ee_key, state_dict[ee_key], ee_dim)
if arm_value.shape[:-1] != ee_value.shape[:-1]:
raise ValueError(
f"'{arm_key}' and '{ee_key}' must share the same prefix shape, "
f"got {arm_value.shape[:-1]} and {ee_value.shape[:-1]}."
)
if expected_prefix_shape is None:
expected_prefix_shape = arm_value.shape[:-1]
elif arm_value.shape[:-1] != expected_prefix_shape:
raise ValueError(
"All state fields must share the same prefix shape. "
f"Expected {expected_prefix_shape}, got {arm_value.shape[:-1]} for '{arm_key}'."
)
parts.append(np.concatenate([arm_value, ee_value], axis=-1))
return np.concatenate(parts, axis=-1)
def unpack_robot_state(
packed_state,
action_type: str,
robot_action_dim_info: dict,
source_type: str = "obs",
):
"""
Unpack packed robot state.
Rules:
- source_type='obs':
* ndim must be <= 2
* if ndim == 1: return dict
* if ndim == 2: return list[dict]
- source_type='dataset':
* return dict of arrays directly
Unpacking order:
single-arm:
[arm_0, ee_0]
dual-arm:
[arm_0, ee_0, arm_1, ee_1]
"""
arm_dims, ee_dims, num_arms = _validate_config(action_type, robot_action_dim_info, source_type)
arm_keys, ee_keys = _get_state_keys(action_type, num_arms, source_type)
packed = np.asarray(packed_state)
expected_dim = sum(arm_dims) + sum(ee_dims)
if packed.shape[-1] != expected_dim:
raise ValueError(
f"packed_state last dim mismatch: expected {expected_dim}, got {packed.shape[-1]}."
)
if source_type == "obs":
assert packed.ndim <= 2, (
f"When source_type='obs', packed_state.ndim must be <= 2, got {packed.ndim}."
)
def _unpack_single_action(single_action: np.ndarray) -> dict:
result = {}
offset = 0
for arm_key, ee_key, arm_dim, ee_dim in zip(
arm_keys, ee_keys, arm_dims, ee_dims
):
result[arm_key] = single_action[offset : offset + arm_dim]
offset += arm_dim
result[ee_key] = single_action[offset : offset + ee_dim]
offset += ee_dim
return result
if packed.ndim == 1:
return _unpack_single_action(packed)
return [_unpack_single_action(single_action) for single_action in packed]
result = {}
offset = 0
for arm_key, ee_key, arm_dim, ee_dim in zip(arm_keys, ee_keys, arm_dims, ee_dims):
result[arm_key] = packed[..., offset : offset + arm_dim]
offset += arm_dim
result[ee_key] = packed[..., offset : offset + ee_dim]
offset += ee_dim
return result
def get_robot_action_dim_info(env_cfg_type):
env_cfg = load_yaml(os.path.join(os.path.dirname(__file__), "../../env_cfg", f"{env_cfg_type}.yml"))
robot_name = env_cfg['config']['robot']
robot_action_dim_info = load_json(os.path.join(os.path.dirname(__file__), "../../env_cfg/robot", "_robot_info.json"))[robot_name]
return robot_action_dim_info
def get_batch_size(env_cfg_type):
env_cfg = load_yaml(os.path.join(os.path.dirname(__file__), "../../env_cfg", f"{env_cfg_type}.yml"))
sim_cfg = env_cfg['config']['sim']
sim_info = load_yaml(os.path.join(os.path.dirname(__file__), "../../env_cfg/sim", f"{sim_cfg}.yml"))
return sim_info['scene']['num_envs']
def get_action_dim(env_cfg_type):
env_cfg = load_yaml(os.path.join(os.path.dirname(__file__), "../../env_cfg", f"{env_cfg_type}.yml"))
robot_name = env_cfg['config']['robot']
robot_action_dim_info = load_json(os.path.join(os.path.dirname(__file__), "../../env_cfg/robot", "_robot_info.json"))[robot_name]
return sum(robot_action_dim_info["arm_dim"]) + sum(robot_action_dim_info["ee_dim"])
def _decode_single_image_bit(image_bit):
"""Decode one encoded image buffer into an HWC uint8 RGB array."""
if isinstance(image_bit, np.ndarray) and image_bit.dtype.kind in {"S", "U"}:
image_bit = image_bit.item() if image_bit.ndim == 0 else image_bit.tobytes()
if isinstance(image_bit, str):
image_bit = image_bit.encode("utf-8")
elif isinstance(image_bit, memoryview):
image_bit = image_bit.tobytes()
if isinstance(image_bit, (bytes, bytearray)):
# Fixed-width HDF5 byte columns pad the tail with NUL.
image_bit = image_bit.rstrip(b"\0")
elif isinstance(image_bit, np.ndarray):
image_bit = np.ascontiguousarray(image_bit)
# The returned array is RGB. This is the conclusion for this repo — do not
# "correct" it with the usual "cv2 means BGR" rule. cv2.imencode/imdecode
# only carry channels in the order they were handed in, so the round trip
# is an identity on channel order, and XPolicyLab trajectory files and
# runtime observations are encoded from RGB arrays. Adding a COLOR_BGR2RGB
# swap here (or in any caller) is what actually breaks the channel order.
image = cv2.imdecode(np.frombuffer(image_bit, np.uint8), cv2.IMREAD_COLOR)
if image is None:
raise ValueError(
f"Failed to decode image bits (type={type(image_bit).__name__}, "
f"size={getattr(image_bit, 'size', len(image_bit) if hasattr(image_bit, '__len__') else '?')})."
)
return image
def _decode_image_bit_sequence(image_bits):
frames = []
for index, image_bit in enumerate(image_bits):
try:
frames.append(decode_image_bit(image_bit))
except ValueError as exc:
raise ValueError(f"Frame {index}: {exc}") from exc
if not frames:
return np.zeros((0,), dtype=np.uint8)
return np.stack(frames, axis=0)
def decode_image_bit(image_bits):
"""
Decode encoded image bit stream(s) into uint8 RGB image array(s).
The output is RGB. Treat that as settled and do not apply the usual
"OpenCV returns BGR" rule: XPolicyLab trajectory files and runtime
observations store buffers encoded from RGB arrays, and encode/decode
round trips preserve the channel order they were given. Never add a
COLOR_BGR2RGB after this function to "correct" the output — there is
nothing to correct, and the swap is what breaks the channel order.
Deliberately converting the RGB result to BGR is a different thing and is
allowed where a checkpoint was trained on BGR data; that must be an opt-in
documented in the adapter (see Dexora_1B's `input_color_order`), never a
silent fix applied at the decode site.
Values that are already decoded are returned unchanged, so this function is
safe to call on an observation or trajectory field without knowing whether
the producer encoded it.
Dispatch is on dtype first, then ndim:
- bytes / bytearray / memoryview / str -> one encoded buffer
- ndarray of dtype kind 'S', 'U', 'O' -> sequence of encoded buffers,
or one buffer when 0-d
- uint8 ndarray, ndim == 1 -> one encoded buffer
- uint8 ndarray, ndim == 2 -> (T, N) stack of encoded buffers
- uint8 ndarray, ndim >= 3 -> already decoded, returned as is
- ndarray of any other dtype -> already decoded, returned as is
- list / tuple -> element-wise, stacked on axis 0
Grayscale (H, W) uint8 images are not supported: a 2-D uint8 array is always
read as a stack of encoded buffers.
Raises:
ValueError: if any buffer fails to decode.
"""
if isinstance(image_bits, (bytes, bytearray, memoryview, str)):
return _decode_single_image_bit(image_bits)
if isinstance(image_bits, np.ndarray):
if image_bits.dtype.kind in {"S", "U", "O"}:
if image_bits.ndim == 0:
return _decode_single_image_bit(image_bits.item())
return _decode_image_bit_sequence(image_bits)
if image_bits.dtype == np.uint8:
if image_bits.ndim == 1:
return _decode_single_image_bit(image_bits)
if image_bits.ndim == 2:
return _decode_image_bit_sequence(image_bits)
return image_bits
if isinstance(image_bits, (list, tuple)):
return _decode_image_bit_sequence(image_bits)
return _decode_single_image_bit(image_bits)
# Color fields of one camera in an observation. 'depth' is deliberately absent:
# depth is stored as 16-bit or float data that cv2.IMREAD_COLOR would destroy.
OBS_IMAGE_KEYS = ("color", "colors", "rgb", "image")
_DECODABLE_TYPES = (bytes, bytearray, memoryview, np.ndarray, list, tuple)
def _decode_obs_image(value, camera_name, image_key):
if value is None:
return value
try:
return decode_image_bit(value)
except (ValueError, TypeError) as exc:
raise ValueError(
f"Failed to decode obs['vision']['{camera_name}']['{image_key}']: {exc}"
) from exc
def decode_obs_images(obs):
"""
Decode the encoded color streams of a runtime observation, in place.
The policy server calls this before handing an observation to the model, so
`update_obs` / `update_obs_batch` always receive plain RGB image arrays and
no adapter has to decode anything. Values that are already decoded pass
through untouched, and only the color fields listed in OBS_IMAGE_KEYS are
visited — depth maps, intrinsics, extrinsics and shapes are left alone.
What the model receives is RGB. That is the conclusion; `model.py` must not
swap channels to "undo" an OpenCV BGR convention that does not apply here.
Args:
obs: One observation dict, or a list/tuple of them for batched eval.
Anything else is returned unchanged.
Returns:
The same object, with encoded color fields replaced by decoded arrays.
Raises:
ValueError: if a color field fails to decode, naming the camera.
"""
if isinstance(obs, (list, tuple)):
for single_obs in obs:
decode_obs_images(single_obs)
return obs
if not isinstance(obs, dict):
return obs
vision = obs.get("vision")
if not isinstance(vision, dict):
return obs
for camera_name, camera in vision.items():
if isinstance(camera, dict):
for image_key in OBS_IMAGE_KEYS:
if image_key in camera:
camera[image_key] = _decode_obs_image(
camera[image_key], camera_name, image_key
)
elif isinstance(camera, _DECODABLE_TYPES):
# Layout where vision/<camera> is the image itself.
vision[camera_name] = _decode_obs_image(camera, camera_name, "color")
return obs
def images_encoding(imgs):
"""
JPEG-encode frames for storage in the XPolicyLab layout, padded to a
common length so they fit a fixed-width HDF5 byte column.
Feed this RGB, never BGR and never a channel-swapped array: cv2.imencode
writes channels in the order it is given, so whatever goes in is what
`decode_image_bit` hands back, and the rest of the pipeline assumes RGB.
"""
encode_data = []
padded_data = []
max_len = 0
for i in range(len(imgs)):
success, encoded_image = cv2.imencode(".jpg", imgs[i])
jpeg_data = encoded_image.tobytes()
encode_data.append(jpeg_data)
max_len = max(max_len, len(jpeg_data))
# padding
for i in range(len(imgs)):
padded_data.append(encode_data[i].ljust(max_len, b"\0"))
return encode_data, max_len

View File

@ -0,0 +1,22 @@
#!/bin/bash
# Resolve EVAL_ENV_TYPE to sim, debug, or real_world.
# Empty or unset defaults to sim.
resolve_eval_env_type() {
local raw="${EVAL_ENV_TYPE:-}"
case "${raw}" in
""|sim)
echo "sim"
;;
debug)
echo "debug"
;;
real|real_world)
echo "real_world"
;;
*)
echo "[ERROR] Unknown EVAL_ENV_TYPE: '${raw}' (expected: sim, debug, real)" >&2
return 1
;;
esac
}

View File

@ -0,0 +1,26 @@
{
"piper": {
"arm_dim": [6, 6],
"ee_dim": [1, 1]
},
"arx_x5": {
"arm_dim": [6, 6],
"ee_dim": [1, 1]
},
"piper_x": {
"arm_dim": [6, 6],
"ee_dim": [1, 1]
},
"tianji_marvin_wuji": {
"arm_dim": [7, 7],
"ee_dim": [20, 20]
},
"aloha_agilex": {
"arm_dim": [6, 6],
"ee_dim": [1, 1]
},
"franka": {
"arm_dim": [7, 7],
"ee_dim": [1, 1]
}
}

View File

@ -0,0 +1,36 @@
#!/bin/bash
set -e
eval_batch="${1}"
eval_env_conda_env="${2}"
free_port="${3}"
bench_name="${4}"
task_name="${5}"
env_cfg_type="${6}"
policy_name="${7}"
additional_info="${8}"
root_dir="${9}"
seed="${10}"
env_gpu_id="${11}"
policy_server_ip="${12:-localhost}"
protocol="${13:-ws}"
source "$(conda info --base)/etc/profile.d/conda.sh"
conda deactivate || true
conda activate "${eval_env_conda_env}"
echo -e "\033[34m[CLIENT] Activating Conda environment: ${eval_env_conda_env}\033[0m"
echo -e "\033[34m[CLIENT] Connecting to server ${policy_server_ip}:${free_port}...\033[0m"
echo -e "\033[34m[CLIENT] Watch for green [CONNECTED]; yellow [RECONNECT] means the client is retrying.\033[0m"
export PYTHONPATH="${root_dir}/XPolicyLab:${root_dir}${PYTHONPATH:+:${PYTHONPATH}}"
python "${root_dir}/XPolicyLab/debug_env_client.py" \
--bench_name "${bench_name}" \
--task_name "${task_name}" \
--env_cfg_type "${env_cfg_type}" \
--policy_name "${policy_name}" \
--protocol "${protocol}" \
--host "${policy_server_ip}" \
--port "${free_port}" \
--eval_batch "${eval_batch}"

View File

@ -0,0 +1,67 @@
#!/bin/bash
set -e
eval_batch="${1}"
eval_env_conda_env="${2}"
free_port="${3}"
bench_name="${4}"
task_name="${5}"
env_cfg_type="${6}"
policy_name="${7}"
additional_info="${8}"
root_dir="${9}"
seed="${10}"
env_gpu_id="${11}"
policy_server_ip="${12:-localhost}"
protocol="${13:-ws}"
IFS='|' read -r action_type base_cfg < <(ADDITIONAL_INFO="${additional_info}" python - <<'PY'
import os
fields = {}
for part in os.environ.get("ADDITIONAL_INFO", "").split(","):
if "=" not in part:
continue
key, value = part.split("=", 1)
fields[key.strip()] = value.strip()
print(fields.get("action_type", "") + "|" + fields.get("base_cfg", ""))
PY
)
base_cfg="${base_cfg:-${BASE_CFG:-}}"
echo -e "\033[31m[WARN] Real-world evaluation is not supported in the open-source release; attempting real env client startup anyway.\033[0m" >&2
source "$(conda info --base)/etc/profile.d/conda.sh"
conda deactivate || true
conda activate "${eval_env_conda_env}"
echo -e "\033[34m[CLIENT] Activating Conda environment: ${eval_env_conda_env}\033[0m"
echo -e "\033[34m[CLIENT] Connecting to server ${policy_server_ip}:${free_port} (real env)...\033[0m"
echo -e "\033[34m[CLIENT] Watch for green [CONNECTED]; yellow [RECONNECT] means the client is retrying.\033[0m"
export PYTHONPATH="${root_dir}/src:${root_dir}/XPolicyLab:${root_dir}:${PYTHONPATH:-}"
if [[ -z "${action_type}" ]]; then
echo "[ERROR] EVAL_ENV_TYPE=real requires action_type (via additional_info or --action_type)" >&2
exit 1
fi
if [[ -z "${base_cfg}" ]]; then
echo "[ERROR] EVAL_ENV_TYPE=real requires base_cfg (via additional_info, --base_cfg, or BASE_CFG)" >&2
exit 1
fi
python "${root_dir}/src/task_env/real_env_client.py" \
--dataset_name "${bench_name}" \
--task_name "${task_name}" \
--env_cfg_type "${env_cfg_type}" \
--policy_name "${policy_name}" \
--protocol "${protocol}" \
--host "${policy_server_ip}" \
--port "${free_port}" \
--eval_batch "${eval_batch}" \
--seed "${seed}" \
--action_type "${action_type}" \
--base_cfg "${base_cfg}" \
--additional_info "${additional_info}" \
--root_dir "${root_dir}"

View File

@ -0,0 +1,38 @@
#!/bin/bash
set -e
eval_batch="${1}"
eval_env_conda_env="${2}"
policy_server_port="${3}"
bench_name="${4}"
task_name="${5}"
env_cfg_type="${6}"
policy_name="${7}"
additional_info="${8}"
root_dir="${9}"
seed="${10}"
env_gpu_id="${11}"
policy_server_ip="${12:-localhost}"
protocol="${13:-ws}"
source "$(conda info --base)/etc/profile.d/conda.sh"
conda deactivate || true
conda activate "${eval_env_conda_env}"
echo -e "\033[34m[CLIENT] Activating Conda environment: ${eval_env_conda_env}\033[0m"
echo -e "\033[34m[CLIENT] Connecting to server ${policy_server_ip}:${policy_server_port}...\033[0m"
echo -e "\033[34m[CLIENT] Watch for green [CONNECTED]; yellow [RECONNECT] means the client is retrying.\033[0m"
bash "${root_dir}/scripts/eval_policy.sh" \
--bench_name "${bench_name}" \
--task_name "${task_name}" \
--env_cfg_type "${env_cfg_type}" \
--policy_name "${policy_name}" \
--host "${policy_server_ip}" \
--port "${policy_server_port}" \
--protocol "${protocol}" \
--eval_batch "${eval_batch}" \
--root_dir "${root_dir}" \
--device_id "${env_gpu_id}" \
--additional_info "${additional_info}" \
--seed "${seed}"

View File

@ -0,0 +1,66 @@
#!/bin/bash
set -e
UTILS_DIR="${1}"
yaml_file="${2}"
eval_env_conda_env="${3}"
policy_server_port="${4}"
bench_name="${5}"
task_name="${6}"
env_cfg_type="${7}"
policy_name="${8}"
additional_info="${9}"
ROOT_DIR="${10}"
seed="${11}"
env_gpu_id="${12}"
policy_server_ip="${13:-localhost}"
protocol_override="${14:-}"
# shellcheck source=resolve_eval_env_type.sh
source "${UTILS_DIR}/resolve_eval_env_type.sh"
eval_env_mode="$(resolve_eval_env_type)" || exit 1
read eval_batch yaml_protocol < <(python - <<PY
import yaml
with open("${yaml_file}", "r") as f:
data = yaml.safe_load(f)
print(
str(data.get("eval_batch", False)).lower(),
data.get("protocol", "ws"),
)
PY
)
protocol="${protocol_override:-${yaml_protocol}}"
if [[ -z "${EVAL_ENV_TYPE:-}" ]]; then
echo "[CLIENT] EVAL_ENV_TYPE=(default sim) -> ${eval_env_mode}"
else
echo "[CLIENT] EVAL_ENV_TYPE=${EVAL_ENV_TYPE} -> ${eval_env_mode}"
fi
COMMON_ARGS=(
"${eval_batch}"
"${eval_env_conda_env}"
"${policy_server_port}"
"${bench_name}"
"${task_name}"
"${env_cfg_type}"
"${policy_name}"
"${additional_info}"
"${ROOT_DIR}"
"${seed}"
"${env_gpu_id}"
"${policy_server_ip}"
)
if [[ "${eval_env_mode}" == "debug" ]]; then
bash "${UTILS_DIR}/run_debug_env_client.sh" "${COMMON_ARGS[@]}" "${protocol}"
elif [[ "${eval_env_mode}" == "sim" ]]; then
bash "${UTILS_DIR}/run_sim_env_client.sh" "${COMMON_ARGS[@]}" "${protocol}"
elif [[ "${eval_env_mode}" == "real_world" ]]; then
echo -e "\033[31m[WARN] EVAL_ENV_TYPE=real: real-world evaluation is not supported in the open-source release; continuing to real env client.\033[0m" >&2
bash "${UTILS_DIR}/run_real_env_client.sh" "${COMMON_ARGS[@]}" "${protocol}"
else
echo "[ERROR] Unknown eval env mode: ${eval_env_mode}" >&2
exit 1
fi

View File

@ -0,0 +1,188 @@
#!/bin/bash
# Wait until policy server process is listening on TCP port or the process exits.
# Prefer LISTEN-state checks to avoid websocket handshake errors; fall back to a
# plain TCP connect on minimal images without ss/lsof/netstat.
# Usage: wait_for_policy_server.sh <host> <port> <server_pid> [label] [timeout_sec]
set -euo pipefail
host=${1:?host required}
port=${2:?port required}
pid=${3:?server pid required}
label=${4:-Policy server}
timeout_sec=${5:-360}
RED=$'\033[31m'
GREEN=$'\033[32m'
BLUE=$'\033[34m'
YELLOW=$'\033[33m'
BOLD=$'\033[1m'
RESET=$'\033[0m'
_draw_wait_progress() {
local elapsed=$1
local total=$2
local wait_label=$3
local wait_host=$4
local wait_port=$5
local width=28
local filled=0
local percent=0
local remaining=0
if (( total > 0 )); then
percent=$(( elapsed * 100 / total ))
filled=$(( elapsed * width / total ))
remaining=$(( total - elapsed ))
fi
if (( filled > width )); then
filled=${width}
fi
if (( remaining < 0 )); then
remaining=0
fi
local bar=""
local i
for (( i = 0; i < width; i++ )); do
if (( i < filled )); then
bar+="#"
else
bar+="."
fi
done
printf '\r%s[CONNECTING]%s %s waiting for %s:%s [%s] %3d%% %ss left' \
"${BLUE}${BOLD}" "${RESET}" "${wait_label}" "${wait_host}" "${wait_port}" "${bar}" "${percent}" "${remaining}"
}
_port_is_listening() {
local listen_host=$1
local listen_port=$2
# Minimal containers often lack ss/lsof/netstat. On Linux, /proc/net/tcp*
# gives us the LISTEN state without opening a websocket connection.
if command -v python3 >/dev/null 2>&1; then
python3 - "${listen_host}" "${listen_port}" <<'PY'
import os
import socket
import sys
host = sys.argv[1]
port_hex = f"{int(sys.argv[2]):04X}"
_WILDCARDS = {"00000000", "00000000000000000000000000000000"}
# /proc/net/tcp stores IPv4 addresses as little-endian hex. Resolve the target
# host so a listener bound to a different interface does not false-positive.
host_hex = None
if host not in ("", "*", "0.0.0.0", "::"):
try:
host_hex = socket.inet_aton(socket.gethostbyname(host))[::-1].hex().upper()
except OSError:
host_hex = None # unresolvable; fall back to port-only matching
def _addr_matches(local_addr_hex: str) -> bool:
if host_hex is None or local_addr_hex in _WILDCARDS:
return True
# Exact IPv4 match, or an IPv4-mapped IPv6 entry ending with the same bytes.
return local_addr_hex == host_hex or local_addr_hex.endswith(host_hex)
checked = False
for proc_path in ("/proc/net/tcp", "/proc/net/tcp6"):
if not os.path.exists(proc_path):
continue
checked = True
with open(proc_path, "r", encoding="utf-8") as handle:
next(handle, None)
for line in handle:
parts = line.split()
if len(parts) < 4 or parts[3] != "0A":
continue
local_addr, _, local_port = parts[1].rpartition(":")
if local_port.upper() == port_hex and _addr_matches(local_addr.upper()):
raise SystemExit(0)
raise SystemExit(1 if checked else 2)
PY
proc_status=$?
if (( proc_status == 0 )); then
return 0
fi
if (( proc_status == 1 )); then
return 1
fi
fi
if command -v ss >/dev/null 2>&1; then
ss -ltn "sport = :${listen_port}" 2>/dev/null | grep -q LISTEN
return $?
fi
if command -v lsof >/dev/null 2>&1; then
lsof -nP -iTCP:"${listen_port}" -sTCP:LISTEN >/dev/null 2>&1
return $?
fi
if command -v netstat >/dev/null 2>&1; then
netstat -an 2>/dev/null | grep -Eq "[\.:]${listen_port}[[:space:]].*LISTEN"
return $?
fi
# Last-resort fallback for non-Linux/minimal systems. This may create a
# short-lived TCP connection, but it keeps eval.sh usable when no LISTEN
# inspection tool is available.
if command -v python3 >/dev/null 2>&1; then
python3 - "${listen_host}" "${listen_port}" >/dev/null 2>&1 <<'PY'
import socket
import sys
host = sys.argv[1]
port = int(sys.argv[2])
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(1)
sock.connect((host, port))
PY
return $?
fi
return 1
}
echo -e "${BLUE}${BOLD}[CONNECTING]${RESET} ${label} -> ${host}:${port} (PID=${pid}, timeout=${timeout_sec}s)"
# Carriage-return progress bars flood redirected logs; only animate on a TTY.
stdout_is_tty=0
if [[ -t 1 ]]; then
stdout_is_tty=1
fi
for elapsed in $(seq 0 "${timeout_sec}"); do
if ! kill -0 "${pid}" 2>/dev/null; then
if (( stdout_is_tty )); then
printf '\n' >&2
fi
echo -e "${RED}${BOLD}[ERROR]${RESET} ${label} (PID=${pid}) exited before opening port ${port}." >&2
exit 1
fi
if _port_is_listening "${host}" "${port}"; then
if (( stdout_is_tty )); then
printf '\n'
fi
echo -e "${GREEN}${BOLD}[CONNECTED]${RESET} ${label} ready on ${host}:${port} (PID=${pid})"
exit 0
fi
if (( stdout_is_tty )); then
_draw_wait_progress "${elapsed}" "${timeout_sec}" "${label}" "${host}" "${port}"
elif (( elapsed > 0 && elapsed % 30 == 0 )); then
echo -e "${BLUE}${BOLD}[CONNECTING]${RESET} ${label} still waiting for ${host}:${port} (${elapsed}/${timeout_sec}s)"
fi
if (( elapsed >= timeout_sec )); then
break
fi
sleep 1
done
if (( stdout_is_tty )); then
printf '\n' >&2
fi
echo -e "${RED}${BOLD}[ERROR]${RESET} ${label} timed out after ${timeout_sec}s waiting for port ${port}." >&2
exit 1

View File

@ -0,0 +1,28 @@
#!/bin/bash
set -e
host="$1"
port="$2"
timeout="${3:-30}"
python3 - "$host" "$port" "$timeout" <<'PY'
import socket
import sys
import time
host = sys.argv[1]
port = int(sys.argv[2])
timeout = float(sys.argv[3])
start = time.time()
while time.time() - start < timeout:
try:
with socket.create_connection((host, port), timeout=1):
print(f"[INFO] Port {port} is ready", file=sys.stderr)
sys.exit(0)
except OSError:
time.sleep(0.2)
print(f"[ERROR] Timeout waiting for {host}:{port}", file=sys.stderr)
sys.exit(1)
PY

239
tests/test_contract.py Normal file
View File

@ -0,0 +1,239 @@
"""契约系统单测 (design_v2 §4.5 D15 校验规则 / 派生规格)。
运行: python -m pytest tests/test_contract.py -q (或 python tests/test_contract.py)
图像输入固定 1:1 (224x224, 256 token/路); 非 1:1 比例已移除。
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as C # noqa: E402
def base_contract(**overrides) -> dict:
data = {
"cameras": ["cam1", "cam2", "cam3", "cam4"],
"state": {"dim": 14, "inject": True},
"language": {"enabled": True, "max_token_len": 200},
"action": {"dim": 14, "horizon": 10},
"norm_mode": "min_max",
"key_map": {
"observation.images.cam_high": ["cam1"],
"observation.images.cam_third_view": ["cam2"],
"observation.images.cam_left_wrist": ["cam3"],
"observation.images.cam_right_wrist": ["cam4"],
"observation.state": "state",
"action": "actions",
"task": "prompt",
},
}
data.update(overrides)
return data
def expect_exit(data_or_text, needle: str) -> None:
"""契约违例必须以非 0 退出且报错含 needle (在子进程中断言 SystemExit)。"""
if isinstance(data_or_text, dict):
payload = json.dumps(data_or_text)
else:
payload = data_or_text
code = (
"import sys;sys.path.insert(0, %r);"
"from pi_pipeline import contract;"
"contract.Contract.from_json(sys.stdin.read())"
% str(Path(__file__).resolve().parents[1])
)
result = subprocess.run(
[sys.executable, "-c", code],
input=payload,
capture_output=True,
text=True,
)
assert result.returncode != 0, f"expected non-zero exit, got 0; stdout={result.stdout}"
combined = result.stdout + result.stderr
assert needle in combined, f"error message missing {needle!r}: {combined}"
# ----------------------------------------------------------------------
# 派生规格 (1:1 固定)
# ----------------------------------------------------------------------
def test_derived_specs():
c1 = C.Contract.from_dict(base_contract())
assert c1.per_cam_resolution == (224, 224)
assert c1.tokens_per_cam == 256
assert c1.prefix_tokens == 4 * 256 + 200
# 旧契约带 aspect_ratio: "1:1" 兼容, 其余取值非 0 退出
c_legacy = C.Contract.from_dict(base_contract(aspect_ratio="1:1"))
assert c_legacy.prefix_tokens == 4 * 256 + 200
def test_robotwin_default():
c = C.Contract.robotwin_default()
assert c.cameras == ("cam1", "cam2", "cam3", "cam4")
assert c.vision_map["observation.images.cam_high"] == ("cam1",)
assert c.state_source == "state"
assert c.action_source == "actions"
assert c.prompt_source == "prompt"
assert c.discrete_state_input is True
assert c.use_quantile_norm is False
kw = c.pi0_config_kwargs()
assert kw["image_keys"] == ("cam1", "cam2", "cam3", "cam4")
assert kw["pi05"] is True
def test_model_structure_roundtrip(tmp_path=None):
c = C.Contract.from_dict(base_contract())
structure = c.to_model_structure()
assert structure["derived"]["tokens_per_cam"] == 256
assert structure["derived"]["prefix_tokens"] == 4 * 256 + 200
restored = C.Contract.from_dict(structure["contract"])
assert restored.to_dict() == c.to_dict()
def test_one_to_many_mapping():
"""一对多: 一个外部键映射到多个内部槽 (同图复用, 默认独立增广 R-11)。"""
data = base_contract()
data["cameras"] = ["cam1", "cam2", "cam3", "cam4", "cam5"]
data["key_map"]["observation.images.cam_high"] = ["cam1", "cam5"]
c = C.Contract.from_dict(data)
assert c.vision_map["observation.images.cam_high"] == ("cam1", "cam5")
assert c.num_cameras == 5
assert c.prefix_tokens == 5 * 256 + 200
def test_camera_count_variants():
for n in (1, 2, 4):
data = base_contract()
data["cameras"] = [f"cam{i}" for i in range(1, n + 1)]
keep = {k: v for k, v in data["key_map"].items() if not isinstance(v, list)}
slots = [f"cam{i}" for i in range(1, n + 1)]
keep["observation.images.cam_high"] = slots
data["key_map"] = keep
c = C.Contract.from_dict(data)
assert c.num_cameras == n
# ----------------------------------------------------------------------
# 校验规则 (D15, 违例非 0 退出)
# ----------------------------------------------------------------------
def test_reject_bad_slot_name():
expect_exit(base_contract(cameras=["base_0_rgb"]), "cam<N>")
def test_reject_duplicate_slots():
expect_exit(base_contract(cameras=["cam1", "cam1"]), "duplicate")
def test_reject_bad_aspect():
expect_exit(base_contract(aspect_ratio="21:9"), "aspect_ratio")
def test_reject_removed_aspect_43():
"""非 1:1 比例 (4:3/16:9) 已移除: 旧契约使用时非 0 退出。"""
expect_exit(base_contract(aspect_ratio="4:3"), "1:1")
expect_exit(base_contract(aspect_ratio="16:9"), "1:1")
def test_reject_missing_slot_mapping():
data = base_contract()
del data["key_map"]["observation.images.cam_left_wrist"]
expect_exit(data, "cam3")
def test_reject_multi_source_same_slot():
data = base_contract()
data["key_map"]["observation.images.cam_head"] = ["cam1"]
expect_exit(data, "multiple sources")
def test_reject_bare_string_image_target():
data = base_contract()
data["key_map"]["observation.images.cam_high"] = "cam1"
expect_exit(data, "LIST")
def test_reject_unknown_slot_target():
data = base_contract()
data["key_map"]["observation.images.cam_high"] = ["cam9"]
expect_exit(data, "unknown slot")
def test_reject_multiple_state_mappings():
data = base_contract()
data["key_map"]["state"] = "state"
expect_exit(data, "state")
def test_reject_missing_action_mapping():
data = base_contract()
del data["key_map"]["action"]
expect_exit(data, "action")
def test_reject_invalid_json():
expect_exit("{not json", "not valid JSON")
def test_reject_missing_prompt():
data = base_contract()
del data["key_map"]["task"]
expect_exit(data, "prompt")
# ----------------------------------------------------------------------
# 归一化模式联动 (D4/D19)
# ----------------------------------------------------------------------
def test_norm_mode_vision_only():
c = C.Contract.from_dict(base_contract(norm_mode="vision_only"))
assert c.discrete_state_input is False
def test_norm_mode_quantile():
c = C.Contract.from_dict(base_contract(norm_mode="quantile"))
assert c.use_quantile_norm is True
def test_norm_mode_rejected():
expect_exit(base_contract(norm_mode="l2"), "norm_mode")
# ----------------------------------------------------------------------
# serve 逆映射
# ----------------------------------------------------------------------
def test_inverse_vision_map():
c = C.Contract.robotwin_default()
inverse = c.inverse_vision_map()
assert inverse == {
"cam1": "observation.images.cam_high",
"cam2": "observation.images.cam_third_view",
"cam3": "observation.images.cam_left_wrist",
"cam4": "observation.images.cam_right_wrist",
}
if __name__ == "__main__":
failures = 0
for name, fn in sorted(globals().items()):
if name.startswith("test_") and callable(fn):
try:
fn()
print(f"PASS {name}")
except AssertionError as exc:
failures += 1
print(f"FAIL {name}: {exc}")
except Exception as exc: # noqa: BLE001
failures += 1
print(f"ERROR {name}: {type(exc).__name__}: {exc}")
print(f"\n{'ALL PASS' if failures == 0 else f'{failures} FAILURES'}")
sys.exit(1 if failures else 0)

View File

@ -0,0 +1,161 @@
"""letterbox golden 一致性单测 (design_v2 §4.3, Q2 裁定落地)。
裁定: 全部实现对齐 numpy 版 (openpi_client PIL resize_with_pad) — golden 只定义在
uint8 域; float 域期望值 = uint8 golden 经精确线性变换 x/127.5-1 派生 (pad 0 →
-1.0, 两域自洽)。JAX/torch 设备端实现以 golden 一致性 (容差级) 锁定行为。
覆盖: 多源分辨率 x 三目标比例 (1:1/4:3/16:9), 上采样与下采样分开断言。
运行环境: 镜像内 venv_train (jax + torch + openpi 安装后):
python /app/src/tests/test_letterbox_golden.py
"""
from __future__ import annotations
import sys
import numpy as np
from openpi.shared import image_tools
from openpi_client import image_tools as client_image_tools
# (height, width) 源分辨率: RoboTwin D435 240x320, Large_D455 480x848,
# Large_D405 480x640, 官方目标 224x224
SOURCE_SIZES = [(240, 320), (480, 848), (480, 640), (224, 224)]
# 目标分辨率 = 三比例派生规格 (design_v2 §2)
TARGET_SIZES = [(224, 224), (168, 224), (126, 224)]
# 容差: golden 是 PIL uint8 插值后取整; 设备端是 float 插值。
# float 域派生 golden 的期望差上界 = 0.5 级 (插值核差, 实测 PIL float vs jax/torch
# antialias float 为 0.50 uint8 级) + 0.5 级 (golden 取整量化) = 1 个 uint8 级 =
# 1/127.5; 另加浮点余量 (实测曾因恰压 2/255 线被 1.2e-7 尾数击穿)。
FLOAT_ATOL = 1.0 / 127.5 + 1e-6
UINT8_ATOL = 1
def make_image(height: int, width: int, seed: int = 0) -> np.ndarray:
rng = np.random.default_rng(seed)
return rng.integers(0, 256, size=(height, width, 3), dtype=np.uint8)
def derived_float_golden(uint8_golden: np.ndarray) -> np.ndarray:
"""Q2 裁定: float 期望 = uint8 golden 经 x/127.5-1 精确线性派生。"""
return uint8_golden.astype(np.float32) / 127.5 - 1.0
def test_golden_pil_shapes_and_pad():
"""golden 本体: 输出形状正确, letterbox 黑边 (uint8 pad=0) 居中。"""
for src_h, src_w in SOURCE_SIZES:
image = make_image(src_h, src_w, seed=src_h)
for tgt_h, tgt_w in TARGET_SIZES:
out = client_image_tools.resize_with_pad(image, tgt_h, tgt_w)
assert out.shape == (tgt_h, tgt_w, 3), f"{(src_h, src_w)} -> {(tgt_h, tgt_w)}: {out.shape}"
# 黑边校验: 4:3 源进 16:9 目标时上下有 pad 行, 全 0
ratio = max(src_w / tgt_w, src_h / tgt_h)
rh, rw = int(src_h / ratio), int(src_w / ratio)
if rh < tgt_h:
pad0 = (tgt_h - rh) // 2
assert np.all(out[:pad0] == 0), "top pad rows must be black"
if rw < tgt_w:
pad0 = (tgt_w - rw) // 2
assert np.all(out[:, :pad0] == 0), "left pad cols must be black"
def test_jax_matches_golden_uint8():
import jax.numpy as jnp
for src_h, src_w in SOURCE_SIZES:
image = make_image(src_h, src_w, seed=src_h + 1)
for tgt_h, tgt_w in TARGET_SIZES:
golden = client_image_tools.resize_with_pad(image, tgt_h, tgt_w)
got = np.asarray(image_tools.resize_with_pad(jnp.asarray(image), tgt_h, tgt_w))
diff = np.abs(got.astype(np.int16) - golden.astype(np.int16))
max_diff = int(diff.max())
assert max_diff <= UINT8_ATOL, (
f"jax uint8 vs golden {(src_h, src_w)}->{(tgt_h, tgt_w)}: max diff {max_diff}"
)
def test_torch_matches_golden_uint8():
import torch
for src_h, src_w in SOURCE_SIZES:
image = make_image(src_h, src_w, seed=src_h + 2)
for tgt_h, tgt_w in TARGET_SIZES:
golden = client_image_tools.resize_with_pad(image, tgt_h, tgt_w)
got = image_tools.resize_with_pad_torch(
torch.from_numpy(image).unsqueeze(0).permute(0, 3, 1, 2), tgt_h, tgt_w
)
got = got.permute(0, 2, 3, 1).squeeze(0).numpy()
diff = np.abs(got.astype(np.int16) - golden.astype(np.int16))
max_diff = int(diff.max())
assert max_diff <= UINT8_ATOL, (
f"torch uint8 vs golden {(src_h, src_w)}->{(tgt_h, tgt_w)}: max diff {max_diff}"
)
def test_jax_float_matches_derived_golden():
"""Q2: float 域 JAX 实现 vs uint8 golden 线性派生 (容差级锁定)。"""
import jax.numpy as jnp
for src_h, src_w in SOURCE_SIZES:
image = make_image(src_h, src_w, seed=src_h + 3)
for tgt_h, tgt_w in TARGET_SIZES:
golden = client_image_tools.resize_with_pad(image, tgt_h, tgt_w)
expected = derived_float_golden(golden)
got = np.asarray(
image_tools.resize_with_pad(jnp.asarray(image, dtype=jnp.float32) / 127.5 - 1.0, tgt_h, tgt_w)
)
max_diff = float(np.abs(got - expected).max())
assert max_diff <= FLOAT_ATOL, (
f"jax float vs derived golden {(src_h, src_w)}->{(tgt_h, tgt_w)}: max diff {max_diff:.4f}"
)
def test_torch_float_matches_derived_golden():
import torch
for src_h, src_w in SOURCE_SIZES:
image = make_image(src_h, src_w, seed=src_h + 4)
for tgt_h, tgt_w in TARGET_SIZES:
golden = client_image_tools.resize_with_pad(image, tgt_h, tgt_w)
expected = derived_float_golden(golden)
got = (
image_tools.resize_with_pad_torch(
torch.from_numpy(image).unsqueeze(0).permute(0, 3, 1, 2).float() / 127.5 - 1.0,
tgt_h,
tgt_w,
)
.permute(0, 2, 3, 1)
.squeeze(0)
.numpy()
)
max_diff = float(np.abs(got - expected).max())
assert max_diff <= FLOAT_ATOL, (
f"torch float vs derived golden {(src_h, src_w)}->{(tgt_h, tgt_w)}: max diff {max_diff:.4f}"
)
def test_no_resize_passthrough_exact():
"""同尺寸输入: golden 必须是恒等映射 (bit 级), 训练路径 224x224 数据零损。"""
image = make_image(224, 224, seed=99)
out = client_image_tools.resize_with_pad(image, 224, 224)
assert np.array_equal(out, image), "same-size letterbox must be identity"
if __name__ == "__main__":
failures = 0
for name, fn in sorted(globals().items()):
if name.startswith("test_") and callable(fn):
try:
fn()
print(f"PASS {name}")
except AssertionError as exc:
failures += 1
print(f"FAIL {name}: {exc}")
except ImportError as exc:
print(f"SKIP {name}: missing dependency ({exc})")
except Exception as exc: # noqa: BLE001
failures += 1
print(f"ERROR {name}: {type(exc).__name__}: {exc}")
print(f"\n{'ALL PASS' if failures == 0 else f'{failures} FAILURES'}")
sys.exit(1 if failures else 0)

6
ws_common/__init__.py Normal file
View File

@ -0,0 +1,6 @@
"""自研共享层 (OpenPi_Dev): 与 RoboTwin2_Dev/ws_common 同源同步副本。
OpenPi 侧当前只消费 input_locate / report_utils 两个模块; camera_map 与
config_override 为 RoboTwin 仿真侧专用, 不在本工程复制。两份 ws_common 的
公共文件修改时需双向同步 (构建上下文各自自包含, 闭环维护)。
"""

157
ws_common/input_locate.py Normal file
View File

@ -0,0 +1,157 @@
"""上游条目定位与输入校验 (design_v5 §6.4 / §11 input_locate)。
读取 INPUT_ARTIFACTS_PATH 清单, 按 portId (或 name) 挑出本功能需要的条目,
数据本体在 path (容器内绝对路径, 直接读)。与 DATASET_PATH 超参二选一, 上游
优先; 两者都没有时非 0 退出。本镜像输入统一为挂载进容器的本地目录, 不做任何
网络存储下载: bos:// 等网络引用一律非 0 退出, 错误信息提示把数据集挂载进容器。
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
def _read_upstream_entries() -> list[dict]:
path = os.environ.get("INPUT_ARTIFACTS_PATH", "")
if not path or not os.path.isfile(path):
return []
try:
with open(path, "r", encoding="utf-8") as f:
entries = json.load(f)
except (json.JSONDecodeError, OSError) as exc:
sys.exit(f"[input] cannot read INPUT_ARTIFACTS_PATH={path}: {exc}")
return entries if isinstance(entries, list) else []
def _validate_local_dir(path: Path, label: str) -> Path:
if str(path).startswith("bos://") or "://" in str(path):
sys.exit(
f"[input] {label} path {path} is a network storage reference; this entry "
"accepts only container-local mounted directories. Mount the dataset into "
"the container and provide it via the upstream port or DATASET_PATH."
)
if not path.is_dir():
sys.exit(f"[input] {label} directory does not exist: {path}")
return path
def _read_json_list(env_key: str) -> list[dict]:
path = os.environ.get(env_key, "")
if not path or not os.path.isfile(path):
return []
try:
with open(path, "r", encoding="utf-8") as f:
items = json.load(f)
except (json.JSONDecodeError, OSError) as exc:
sys.exit(f"[input] cannot read {env_key}={path}: {exc}")
return items if isinstance(items, list) else []
def _binding_source_node(port_id: str) -> str:
"""平台契约: input-bindings.json 记录 targetPortId -> sourceNodeId 映射."""
for b in _read_json_list("INPUT_BINDINGS_PATH"):
if b.get("targetPortId") == port_id:
return str(b.get("sourceNodeId") or "").strip()
return ""
def _resolve_upstream_path(raw: str, entry: dict, port_id: str) -> str:
"""相对路径按平台约定解析: 上游产物 path 相对其产出节点 outputs 目录,
下游经 $WORKSPACE_ROOT/nodes/<sourceNodeId>/outputs/<path> 访问
(sourceNodeId 取 artifacts 条目 nodeId, 缺失时回退 input-bindings 的
sourceNodeId; 再兜底 $WORKSPACE_ROOT/nodes/*/outputs 下唯一匹配)。"""
if raw.startswith("/") or "://" in raw:
return raw
ws_root = os.environ.get("WORKSPACE_ROOT", "").strip()
if not ws_root:
return raw
node_id = str(entry.get("nodeId") or "").strip() or _binding_source_node(port_id)
if node_id:
return os.path.join(ws_root, "nodes", node_id, "outputs", raw)
import glob
hits = glob.glob(os.path.join(ws_root, "nodes", "*", "outputs", raw))
if len(hits) == 1:
return hits[0]
return raw
def is_lerobot_dataset_root(path: Path) -> bool:
"""LeRobot v3 数据集根的判据: meta/info.json 存在。"""
return (path / "meta" / "info.json").is_file()
def stage_to_local(root: Path) -> Path:
"""平台 workspace (网络 PVC, bosfs) 上的输入无条件暂存到节点本地盘。
训练随机读 PVC 实测慢 4-5 倍 (norm_stats ~29s/500 帧), 且多进程并发读网络盘
不稳定; 输入根位于 $WORKSPACE_ROOT 下时复制到 /tmp/pi-stage/<pid>/ (per-PID,
避免 torch DDP / jax 多 rank 竞态), 本地直跑场景 (不在 workspace 下) 原样返回。
与被否决的 PI_DATA_LOCAL 环境开关不同: 这是平台形态的内生行为, 不暴露超参。
"""
ws_root = os.environ.get("WORKSPACE_ROOT", "").strip()
if not ws_root:
return root
try:
root.resolve().relative_to(Path(ws_root).resolve())
except ValueError:
return root # 不在平台 workspace 下 (本地直跑), 不暂存
import shutil
dst_root = Path("/tmp/pi-stage") / str(os.getpid())
dst_root.mkdir(parents=True, exist_ok=True)
dst = dst_root / root.name
if dst.exists():
shutil.rmtree(dst)
print(f"[input] staging PVC input to node-local disk: {root} -> {dst}", flush=True)
shutil.copytree(root, dst)
print(f"[input] staging done ({dst})", flush=True)
return dst
def expand_dataset_roots(root: Path) -> list[Path]:
"""输入根分类 (多数据集聚合入口, 打破 D17 单数据集边界):
- 根自身是数据集 (有 meta/info.json) -> [根] (单数据集, 行为不变)
- 根是父目录 -> 其下每个含 meta/info.json 的子目录都是一个数据集,
按目录名排序返回 (确定性顺序); 一个都没有则非 0 退出
单端口承载多数据集 (平台一端口一上游约束下, 多数据集 = 一个父目录产物),
多数据集必须共父目录 (HF_LEROBOT_HOME 指向共同父目录, repo_id = 目录名)。
"""
if is_lerobot_dataset_root(root):
return [root]
subs = sorted(p for p in root.iterdir() if p.is_dir() and is_lerobot_dataset_root(p))
if not subs:
sys.exit(
f"[input] {root} is neither a LeRobot v3 dataset root (no meta/info.json) "
f"nor a parent of datasets; contents: {sorted(p.name for p in root.iterdir())[:10]}"
)
print(f"[input] multi-dataset root: {len(subs)} datasets under {root}: "
+ ", ".join(p.name for p in subs), flush=True)
return subs
def locate_input_dir(port_id: str = "dataset", env_key: str = "DATASET_PATH") -> Path:
"""上游端口 > 环境变量超参; 至少其一, 缺失非 0 退出。返回本地目录 Path。
返回的是输入根 (可能是单数据集根或多数据集父目录), 调用方用
expand_dataset_roots 展开成数据集列表。"""
for entry in _read_upstream_entries():
if entry.get("portId") == port_id or entry.get("name") == port_id:
raw = entry.get("path")
if not raw:
sys.exit(f"[input] upstream entry portId={port_id} has empty path")
return _validate_local_dir(Path(_resolve_upstream_path(raw, entry, port_id)), f"upstream port '{port_id}'")
value = os.environ.get(env_key, "").strip()
if value:
return _validate_local_dir(Path(value), env_key)
sys.exit(
f"[input] no input provided: neither upstream port '{port_id}' "
f"nor {env_key} is set. Provide at least one."
)

60
ws_common/report_utils.py Normal file
View File

@ -0,0 +1,60 @@
"""产物清单与进度标记 (design_v5 §11 report_utils / §12 收尾协议)。
- write_artifacts: 业务收尾时把产物清单 (name/type/portId/path/sizeBytes 实测)
写入 $OUTPUT_DIR/.artifacts.json, report.sh 读取它拼入 RESULT_OUTPUT_PATH 的
artifacts 字段。portId 与注册侧 outputs 端口 id 一致。
- emit_progress: 长任务逐 episode/逐文件打印 DROBOTICFLOW_PROGRESS + JSON
(含 RFC3339 updatedAt), 平台监控页取最后一条。
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from pathlib import Path
def _rfc3339_now() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def emit_progress(stage: str, **fields) -> None:
payload = {"stage": stage}
payload.update(fields)
payload["updatedAt"] = _rfc3339_now()
print("DROBOTICFLOW_PROGRESS " + json.dumps(payload, ensure_ascii=False), flush=True)
def _path_size(path: Path) -> int:
if path.is_file():
return path.stat().st_size
total = 0
for root, _dirs, files in os.walk(path):
for name in files:
try:
total += (Path(root) / name).stat().st_size
except OSError:
pass
return total
def write_artifacts(output_dir, artifacts: list[dict]) -> None:
"""artifacts 每项: {"name","type","portId","path"}; sizeBytes 实测补齐。"""
output_dir = Path(output_dir)
resolved = []
for item in artifacts:
path = Path(item["path"])
size = _path_size(path) if path.exists() else 0
resolved.append(
{
"name": item["name"],
"type": item.get("type", "file"),
"portId": item["portId"],
"path": str(path),
"sizeBytes": size,
}
)
target = output_dir / ".artifacts.json"
target.write_text(json.dumps(resolved, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"[report] artifacts manifest written: {target} ({len(resolved)} items)")