OpenPi_POC_Dev/patches/openpi_da7ce8b.patch
2026-10-09 16:41:13 +08:00

926 lines
46 KiB
Diff

From da7ce8b6720afdbd28054a3ca9d804eb60c9f9f8 Mon Sep 17 00:00:00 2001
From: "chao01.wu" <chao01.wu@local>
Date: Tue, 8 Sep 2026 21:28:01 +0800
Subject: [PATCH] DataPipeline: 1:1-only revert (aspect/pos_emb slice removed,
siglip+modeling_siglip restored to official) + multi-dataset concat
(DataConfig.extra_repo_ids, create_torch_dataset Concat)
---
examples/convert_jax_model_to_pytorch.py | 6 +-
scripts/train_pytorch.py | 19 ++-
src/openpi/models/gemma.py | 13 +-
src/openpi/models/model.py | 40 ++++++-
src/openpi/models/pi0.py | 40 ++++++-
src/openpi/models/pi0_config.py | 25 ++--
src/openpi/models_pytorch/pi0_pytorch.py | 10 +-
.../models_pytorch/preprocessing_pytorch.py | 5 +
src/openpi/shared/array_typing.py | 22 ++--
src/openpi/shared/image_tools.py | 10 +-
src/openpi/shared/nnx_utils.py | 8 +-
src/openpi/shared/normalize.py | 7 +-
src/openpi/training/checkpoints.py | 8 +-
src/openpi/training/config.py | 8 ++
src/openpi/training/data_loader.py | 59 +++++++--
src/openpi/training/sharding.py | 75 ++++++++++--
src/openpi/training/utils.py | 6 +
src/openpi/transforms.py | 112 +++++++++++++++---
18 files changed, 402 insertions(+), 71 deletions(-)
diff --git a/examples/convert_jax_model_to_pytorch.py b/examples/convert_jax_model_to_pytorch.py
index 632c0b8..e4459bb 100644
--- a/examples/convert_jax_model_to_pytorch.py
+++ b/examples/convert_jax_model_to_pytorch.py
@@ -290,7 +290,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
llm_mlp_linear = state_dict.pop(f"llm/layers/mlp_{num_expert}/linear{suffix}")
# Check if we have Dense layers (for pi05/adaptive normalization) or scale layers (for regular pi0)
- if "pi05" in checkpoint_dir:
+ if pi05:
# Pi05 with adaptive normalization
llm_input_layernorm_bias = state_dict.pop(f"llm/layers/pre_attention_norm_{num_expert}/Dense_0/bias{suffix}")
llm_post_attention_layernorm_bias = state_dict.pop(f"llm/layers/pre_ffw_norm_{num_expert}/Dense_0/bias{suffix}")
@@ -345,7 +345,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
i
].transpose()
- if "pi05" in checkpoint_dir:
+ if pi05:
# Pi05 with adaptive normalization - use Dense layer parameters directly
state_dict[f"paligemma_with_expert.gemma_expert.model.layers.{i}.input_layernorm.dense.bias"] = (
llm_input_layernorm_bias[i]
@@ -369,7 +369,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
)
# Handle final norm layer
- if "pi05" in checkpoint_dir:
+ if pi05:
# Pi05 with adaptive normalization - use Dense layer parameters directly
final_norm_bias = state_dict.pop(f"llm/final_norm_{num_expert}/Dense_0/bias{suffix}")
final_norm_kernel = state_dict.pop(f"llm/final_norm_{num_expert}/Dense_0/kernel{suffix}")
diff --git a/scripts/train_pytorch.py b/scripts/train_pytorch.py
index c7ddd2b..fa698ca 100644
--- a/scripts/train_pytorch.py
+++ b/scripts/train_pytorch.py
@@ -455,8 +455,25 @@ def train_loop(config: _config.TrainConfig):
end_lr = config.lr_schedule.decay_lr
# Create optimizer with config parameters
+ # DataPipeline 改动 (design_v2 §5.3 注记): FREEZE_BACKBONE=1 冻结 Paligemma 主干
+ # (vision/llm/expert), 只训练 action_in_proj/action_out_proj/time_mlp 头 —
+ # 32GB 卡全参 AdamW (params+grads+opt 状态 ~36GB fp32) 不可行时的低显存路径
+ # (官方 torch 栈无 LoRA 支持, 冒烟/验收语义为可训练可转化, 不要求全参收敛)
+ trainable_params = model.parameters()
+ if os.environ.get("FREEZE_BACKBONE", "").strip().lower() in ("1", "true", "yes"):
+ frozen, trainable = 0, 0
+ for name, p in model.named_parameters():
+ if any(head in name for head in ("action_in_proj", "action_out_proj", "time_mlp")):
+ p.requires_grad = True
+ trainable += p.numel()
+ else:
+ p.requires_grad = False
+ frozen += p.numel()
+ trainable_params = [p for p in model.parameters() if p.requires_grad]
+ logging.info(f"[freeze] trainable={trainable/1e6:.1f}M frozen={frozen/1e6:.1f}M params")
+
optim = torch.optim.AdamW(
- model.parameters(),
+ trainable_params,
lr=peak_lr,
betas=(config.optimizer.b1, config.optimizer.b2),
eps=config.optimizer.eps,
diff --git a/src/openpi/models/gemma.py b/src/openpi/models/gemma.py
index d1623d1..7fbb113 100644
--- a/src/openpi/models/gemma.py
+++ b/src/openpi/models/gemma.py
@@ -146,7 +146,18 @@ class Embedder(nn.Module):
)
def encode(self, x):
- x = self.input_embedding_table[(x,)]
+ # DataPipeline 改动: jax>=0.11 下词表 gather (replicated embedding 表 x
+ # batch 分片 indices) 无法自动解析输出 sharding (模型内 prompt 与 state
+ # 离散 token concat 后继承 batch 分片)。按 jax 官方姿势显式声明输出
+ # 分片 = indices 各维 + embedding 维, 维度 0 对齐 batch 轴。
+ mesh = sharding.active_mesh()
+ if mesh is not None and mesh.devices.size > 1:
+ spec = jax.sharding.PartitionSpec(sharding.BATCH_AXIS, *((None,) * (x.ndim - 1)))
+ x = self.input_embedding_table.at[x].get(
+ out_sharding=jax.sharding.NamedSharding(mesh, spec)
+ )
+ else:
+ x = self.input_embedding_table[(x,)]
x *= jnp.sqrt(self.embed_dim).astype(x.dtype)
return x
diff --git a/src/openpi/models/model.py b/src/openpi/models/model.py
index 29618b4..073e802 100644
--- a/src/openpi/models/model.py
+++ b/src/openpi/models/model.py
@@ -3,6 +3,7 @@ from collections.abc import Sequence
import dataclasses
import enum
import logging
+import os
import pathlib
from typing import Generic, TypeVar
@@ -81,6 +82,10 @@ IMAGE_RESOLUTION = (224, 224)
@at.typecheck
@struct.dataclass
class Observation(Generic[ArrayT]):
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) 同 TrainState — __no_type_check__ 跳过 jaxtyped dataclass __init__
+ # 检查 (jaxtyping>=0.11 tracing 下 images dict 以 None 占位初始化被误拒)
+ __no_type_check__ = True
"""Holds observations, i.e., inputs to the model.
See `Observation.from_dict` to see the expected dictionary form. This is the format
@@ -141,6 +146,20 @@ class Observation(Generic[ArrayT]):
Actions = at.Float[ArrayT, "*b ah ad"]
+def _reshard_to_data_axis(x):
+ """DataPipeline 改动: jax>=0.11 vmap/concat 的 sharding 一致性辅助。
+
+ 多进程 mesh 下把数组 reshard 到数据轴 (与 training.sharding.data_axis_spec
+ 同语义: fsdp=1 时退化单 batch 轴); 无 mesh 上下文 (单进程推理/转换自检) 时
+ 原样返回。
+ """
+ mesh = jax.sharding.get_abstract_mesh()
+ if not mesh.empty and mesh.size > 1:
+ axis = ("batch",) if mesh.shape["fsdp"] == 1 else ("batch", "fsdp")
+ x = jax.sharding.reshard(x, jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(axis)))
+ return x
+
+
def preprocess_observation(
rng: at.KeyArrayLike | None,
observation: Observation,
@@ -165,7 +184,11 @@ def preprocess_observation(
logger.info(f"Resizing image {key} from {image.shape[1:3]} to {image_resolution}")
image = image_tools.resize_with_pad(image, *image_resolution)
- if train:
+ if train and os.environ.get("OPENPI_DISABLE_AUGMENT", "").strip().lower() not in ("1", "true", "yes"):
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) OPENPI_DISABLE_AUGMENT=1 跳过 augmax 增广 — augmax (jax 0.4
+ # 时代库) 与 jax>=0.11 不兼容 (ColorJitter pixelwise 内部崩, 风险项
+ # R-13), 冒烟/推理不受影响; 正式训练前需替换或修复增广实现
# Convert from [-1, 1] to [0, 1] for augmax.
image = image / 2.0 + 0.5
@@ -181,6 +204,12 @@ def preprocess_observation(
augmax.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5),
]
sub_rngs = jax.random.split(rng, image.shape[0])
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) jax>=0.11 要求 vmap 消去维度的 sharding 一致且全 Explicit mesh
+ # 下 with_sharding_constraint 是 assert — 用 reshard 把 split 产生的
+ # replicated sub_rngs 真分片到数据轴 (与 image 一致); 无 mesh 上下文
+ # (单进程推理/转换自检) 时跳过
+ sub_rngs = _reshard_to_data_axis(sub_rngs)
image = jax.vmap(augmax.Chain(*transforms))(sub_rngs, image)
# Back to [-1, 1].
@@ -312,7 +341,14 @@ def restore_params(
with ocp.PyTreeCheckpointer() as ckptr:
metadata = ckptr.metadata(params_path)
- item = {"params": metadata["params"]}
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
+ # 改动点: 1) orbax>=0.12 的 metadata() 返回 StepMetadata (树结构在
+ # item_metadata["params"]), 旧版直接下标; 双形态兼容
+ if hasattr(metadata, "item_metadata"):
+ params_metadata = metadata.item_metadata["params"]
+ else:
+ params_metadata = metadata["params"]
+ item = {"params": params_metadata}
params = ckptr.restore(
params_path,
diff --git a/src/openpi/models/pi0.py b/src/openpi/models/pi0.py
index ae7c459..09dddda 100644
--- a/src/openpi/models/pi0.py
+++ b/src/openpi/models/pi0.py
@@ -11,6 +11,7 @@ from openpi.models import model as _model
from openpi.models import pi0_config
import openpi.models.gemma as _gemma
import openpi.models.siglip as _siglip
+import openpi.training.sharding as _sharding
from openpi.shared import array_typing as at
logger = logging.getLogger("openpi")
@@ -67,6 +68,9 @@ class Pi0(_model.BaseModel):
def __init__(self, config: pi0_config.Pi0Config, rngs: nnx.Rngs):
super().__init__(config.action_dim, config.action_horizon, config.max_token_len)
self.pi05 = config.pi05
+ # DataPipeline 改动 (D15/D2): 保存 config 引用, preprocess_observation 与
+ # SigLIP 构造从契约派生字段取图像槽位/分辨率/pos_emb 行切片。
+ self.config = config
paligemma_config = _gemma.get_config(config.paligemma_variant)
action_expert_config = _gemma.get_config(config.action_expert_variant)
# TODO: rewrite gemma in NNX. For now, use bridge.
@@ -177,7 +181,12 @@ class Pi0(_model.BaseModel):
action_expert_tokens = action_time_tokens
adarms_cond = None
tokens.append(action_expert_tokens)
- input_mask.append(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
+ # 来源: openpi仓库 src/openpi/models/pi0.py @ 15a9616
+ # 改动点: 1) jax>=0.11 要求 concat 操作数 sharding 一致 — jit 内 jnp.ones 生成的
+ # suffix_mask 是 replicated, 与来自数据分片的 prefix_mask 冲突; 约束到数据轴
+ # (无 mesh 上下文时跳过)
+ suffix_mask = _model._reshard_to_data_axis(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
+ input_mask.append(suffix_mask)
# image/language/state inputs do not attend to action tokens
ar_mask += [True] + ([False] * (self.action_horizon - 1))
tokens = jnp.concatenate(tokens, axis=1)
@@ -190,9 +199,20 @@ class Pi0(_model.BaseModel):
self, rng: at.KeyArrayLike, observation: _model.Observation, actions: _model.Actions, *, train: bool = False
) -> at.Float[at.Array, "*b ah"]:
preprocess_rng, noise_rng, time_rng = jax.random.split(rng, 3)
- observation = _model.preprocess_observation(preprocess_rng, observation, train=train)
+ # DataPipeline 改动 (D15): 图像槽位/分辨率取契约派生字段 (默认官方值)。
+ observation = _model.preprocess_observation(
+ preprocess_rng,
+ observation,
+ train=train,
+ image_keys=self.config.resolved_image_keys,
+ )
batch_shape = actions.shape[:-2]
+ # DataPipeline 改动: jax>=0.11 下不同子路径 (图像/语言 vs actions/time)
+ # 的自然分片可能物理等价但 spec 不同 (batch 轴份额 1 时 P('batch') 与
+ # P(('batch','fsdp')) 并存), concatenate 严格比较即报错; 入口统一
+ # reshard 到维度 0 对齐 batch 轴
+ observation = _sharding.activation_sharding_constraint(observation)
noise = jax.random.normal(noise_rng, actions.shape)
time = jax.random.beta(time_rng, 1.5, 1, batch_shape) * 0.999 + 0.001
time_expanded = time[..., None, None]
@@ -202,6 +222,14 @@ class Pi0(_model.BaseModel):
# one big forward pass of prefix + suffix at once
prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation)
suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(observation, x_t, time)
+ # DataPipeline 改动: prefix (observation 路径) 与 suffix (actions/time 路径)
+ # 的自然分片在 batch 轴份额 1 时会得到 P('batch') 与 P(('batch','fsdp'))
+ # 两种物理等价但 spec 不同的布局, concatenate 严格比较报错; 统一 reshard
+ _unify = _sharding.activation_sharding_constraint
+ prefix_tokens, prefix_mask, prefix_ar_mask = _unify((prefix_tokens, prefix_mask, prefix_ar_mask))
+ suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = _unify(
+ (suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond)
+ )
input_mask = jnp.concatenate([prefix_mask, suffix_mask], axis=1)
ar_mask = jnp.concatenate([prefix_ar_mask, suffix_ar_mask], axis=0)
attn_mask = make_attn_mask(input_mask, ar_mask)
@@ -222,7 +250,13 @@ class Pi0(_model.BaseModel):
num_steps: int | at.Int[at.Array, ""] = 10,
noise: at.Float[at.Array, "b ah ad"] | None = None,
) -> _model.Actions:
- observation = _model.preprocess_observation(None, observation, train=False)
+ # DataPipeline 改动 (D15): 同 compute_loss, 契约派生槽位/分辨率。
+ observation = _model.preprocess_observation(
+ None,
+ observation,
+ train=False,
+ image_keys=self.config.resolved_image_keys,
+ )
# note that we use the convention more common in diffusion literature, where t=1 is noise and t=0 is the target
# distribution. yes, this is the opposite of the pi0 paper, and I'm sorry.
dt = -1.0 / num_steps
diff --git a/src/openpi/models/pi0_config.py b/src/openpi/models/pi0_config.py
index 584d83f..078c628 100644
--- a/src/openpi/models/pi0_config.py
+++ b/src/openpi/models/pi0_config.py
@@ -32,6 +32,11 @@ class Pi0Config(_model.BaseModelConfig):
# This config option is not used directly by the model, but it is read by the ModelTransformFactory.
discrete_state_input: bool = None # type: ignore
+ # DataPipeline 改动 (design_v2 D15, 契约驱动): 相机槽位。
+ # 默认 None = 官方行为 (model.IMAGE_KEYS 3 槽, 224x224, 完整 16x16 网格), 向后兼容。
+ # 契约注入示例: image_keys=("cam1","cam2","cam3","cam4")
+ image_keys: tuple[str, ...] | None = None
+
pytorch_compile_mode: str | None = "max-autotune"
def __post_init__(self):
@@ -47,6 +52,11 @@ class Pi0Config(_model.BaseModelConfig):
"max-autotune-no-cudagraphs",
]
+ @property
+ def resolved_image_keys(self) -> tuple[str, ...]:
+ """契约槽位或官方默认 (cam 序号即 prefix token 拼接顺序, D15)。"""
+ return self.image_keys if self.image_keys else _model.IMAGE_KEYS
+
@property
@override
def model_type(self) -> _model.ModelType:
@@ -62,21 +72,16 @@ class Pi0Config(_model.BaseModelConfig):
@override
def inputs_spec(self, *, batch_size: int = 1) -> tuple[_model.Observation, _model.Actions]:
+ # DataPipeline 改动 (D15): 图像槽位由契约派生 (resolved_image_keys),
+ # 默认与官方一致 (3 槽 224x224)。
image_spec = jax.ShapeDtypeStruct([batch_size, *_model.IMAGE_RESOLUTION, 3], jnp.float32)
image_mask_spec = jax.ShapeDtypeStruct([batch_size], jnp.bool_)
+ keys = self.resolved_image_keys
with at.disable_typechecking():
observation_spec = _model.Observation(
- images={
- "base_0_rgb": image_spec,
- "left_wrist_0_rgb": image_spec,
- "right_wrist_0_rgb": image_spec,
- },
- image_masks={
- "base_0_rgb": image_mask_spec,
- "left_wrist_0_rgb": image_mask_spec,
- "right_wrist_0_rgb": image_mask_spec,
- },
+ images={key: image_spec for key in keys},
+ image_masks={key: image_mask_spec for key in keys},
state=jax.ShapeDtypeStruct([batch_size, self.action_dim], jnp.float32),
tokenized_prompt=jax.ShapeDtypeStruct([batch_size, self.max_token_len], jnp.int32),
tokenized_prompt_mask=jax.ShapeDtypeStruct([batch_size, self.max_token_len], bool),
diff --git a/src/openpi/models_pytorch/pi0_pytorch.py b/src/openpi/models_pytorch/pi0_pytorch.py
index e68ddb7..f8dee9d 100644
--- a/src/openpi/models_pytorch/pi0_pytorch.py
+++ b/src/openpi/models_pytorch/pi0_pytorch.py
@@ -160,8 +160,14 @@ class PI0Pytorch(nn.Module):
return torch.where(att_2d_masks_4d, 0.0, -2.3819763e38)
def _preprocess_observation(self, observation, *, train=True):
- """Helper method to preprocess observation."""
- observation = _preprocessing.preprocess_observation_pytorch(observation, train=train)
+ """Helper model to preprocess observation."""
+ # DataPipeline 改动 (design_v2 D15): 图像槽位取契约派生字段
+ # (Pi0Config.resolved_image_keys, 默认官方值), 与 JAX 侧一致。
+ observation = _preprocessing.preprocess_observation_pytorch(
+ observation,
+ train=train,
+ image_keys=self.config.resolved_image_keys,
+ )
return (
list(observation.images.values()),
list(observation.image_masks.values()),
diff --git a/src/openpi/models_pytorch/preprocessing_pytorch.py b/src/openpi/models_pytorch/preprocessing_pytorch.py
index 33c94a5..d53cbc7 100644
--- a/src/openpi/models_pytorch/preprocessing_pytorch.py
+++ b/src/openpi/models_pytorch/preprocessing_pytorch.py
@@ -39,6 +39,11 @@ def preprocess_observation_pytorch(
# TODO: This is a hack to handle both [B, C, H, W] and [B, H, W, C] formats
# Handle both [B, C, H, W] and [B, H, W, C] formats
+ # DataPipeline 改动: HWC 输入 (B,H,W,C: 尾维==3 且 shape[1]!=3) 先归一到
+ # CHW — 官方 torch 数据管线为 CHW, 但 LeRobot 侧共用 transform 链为 HWC;
+ # 下游 embed_image -> transformers get_image_features 只接受 CHW
+ if image.shape[-1] == 3 and image.shape[1] != 3:
+ image = image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
is_channels_first = image.shape[1] == 3 # Check if channels are in dimension 1
if is_channels_first:
diff --git a/src/openpi/shared/array_typing.py b/src/openpi/shared/array_typing.py
index 569eafe..0be9e74 100644
--- a/src/openpi/shared/array_typing.py
+++ b/src/openpi/shared/array_typing.py
@@ -26,21 +26,25 @@ import torch
# the problem is that custom PyTree nodes are sometimes initialized with arbitrary types (e.g., `jax.ShapeDtypeStruct`,
# `jax.Sharding`, or even <object>) due to JAX tracing operations. this patch skips typechecking when the stack trace
# contains `jax._src.tree_util`, which should only be the case during tree unflattening.
-_original_check_dataclass_annotations = jaxtyping._decorator._check_dataclass_annotations # noqa: SLF001
+# 来源: openpi仓库 src/openpi/shared/array_typing.py @ 15a9616
+# 改动点: 1) jaxtyping>=0.3 移除了 _check_dataclass_annotations 私有 API (issue #277 已上游修复),
+# 改为存在性判断: 旧版打 patch, 新版跳过
+_original_check_dataclass_annotations = getattr(jaxtyping._decorator, "_check_dataclass_annotations", None) # noqa: SLF001
# Redefine Array to include both JAX arrays and PyTorch tensors
Array = jax.Array | torch.Tensor
-def _check_dataclass_annotations(self, typechecker):
- if not any(
- frame.frame.f_globals.get("__name__") in {"jax._src.tree_util", "flax.nnx.transforms.compilation"}
- for frame in inspect.stack()
- ):
- return _original_check_dataclass_annotations(self, typechecker)
- return None
+if _original_check_dataclass_annotations is not None:
+ def _check_dataclass_annotations(self, typechecker):
+ if not any(
+ frame.frame.f_globals.get("__name__") in {"jax._src.tree_util", "flax.nnx.transforms.compilation"}
+ for frame in inspect.stack()
+ ):
+ return _original_check_dataclass_annotations(self, typechecker)
+ return None
-jaxtyping._decorator._check_dataclass_annotations = _check_dataclass_annotations # noqa: SLF001
+ jaxtyping._decorator._check_dataclass_annotations = _check_dataclass_annotations # noqa: SLF001
KeyArrayLike: TypeAlias = jax.typing.ArrayLike
Params: TypeAlias = PyTree[Float[ArrayLike, "..."]]
diff --git a/src/openpi/shared/image_tools.py b/src/openpi/shared/image_tools.py
index 8cde353..6c5c034 100644
--- a/src/openpi/shared/image_tools.py
+++ b/src/openpi/shared/image_tools.py
@@ -90,8 +90,16 @@ def resize_with_pad_torch(
resized_width = int(cur_width / ratio)
# Resize
+ # 来源: openpi仓库 src/openpi/shared/image_tools.py @ 15a9616
+ # 改动点: 1) bilinear 时开 antialias=True, 对齐 golden (PIL BILINEAR 缩小时为抗锯齿
+ # scale-aware 三角核; jax.image.resize 默认 antialias=True 天然一致,
+ # F.interpolate 默认 False 导致缩小场景与 golden 大范围偏差)
resized_images = F.interpolate(
- images, size=(resized_height, resized_width), mode=mode, align_corners=False if mode == "bilinear" else None
+ images,
+ size=(resized_height, resized_width),
+ mode=mode,
+ align_corners=False if mode == "bilinear" else None,
+ antialias=True if mode == "bilinear" else False,
)
# Handle dtype-specific clipping
diff --git a/src/openpi/shared/nnx_utils.py b/src/openpi/shared/nnx_utils.py
index 29df222..5ea9bf9 100644
--- a/src/openpi/shared/nnx_utils.py
+++ b/src/openpi/shared/nnx_utils.py
@@ -65,5 +65,9 @@ class PathRegex:
def state_map(state: nnx.State, filter: nnx.filterlib.Filter, fn: Callable[[Any], Any]) -> nnx.State:
"""Apply a function to the leaves of the state that match the filter."""
- filtered_keys = set(state.filter(filter).flat_state())
- return state.map(lambda k, v: fn(v) if k in filtered_keys else v)
+ # 来源: openpi仓库 src/openpi/shared/nnx_utils.py @ 15a9616
+ # 改动点: 1) flax>=0.12 的 State.flat_state() 返回 FlatState 序列 (旧版 dict),
+ # set(...flat_state()) 对 Param 不可 hash 报错; 改为 to_predicate 逐叶判断,
+ # 两版语义一致
+ predicate = nnx.filterlib.to_predicate(filter)
+ return state.map(lambda k, v: fn(v) if predicate(tuple(k), v) else v)
diff --git a/src/openpi/shared/normalize.py b/src/openpi/shared/normalize.py
index 5a75fae..f56daea 100644
--- a/src/openpi/shared/normalize.py
+++ b/src/openpi/shared/normalize.py
@@ -12,6 +12,10 @@ class NormStats:
std: numpydantic.NDArray
q01: numpydantic.NDArray | None = None # 1st quantile
q99: numpydantic.NDArray | None = None # 99th quantile
+ # DataPipeline 改动 (design_v2 D4): 真 min/max 统计, 供 min_max 归一化模式。
+ # 旧 norm_stats.json 缺字段时为 None, 消费方回退直方图分位数并 INFO。
+ min: numpydantic.NDArray | None = None
+ max: numpydantic.NDArray | None = None
class RunningStats:
@@ -83,7 +87,8 @@ class RunningStats:
variance = self._mean_of_squares - self._mean**2
stddev = np.sqrt(np.maximum(0, variance))
q01, q99 = self._compute_quantiles([0.01, 0.99])
- return NormStats(mean=self._mean, std=stddev, q01=q01, q99=q99)
+ # DataPipeline 改动 (D4): 暴露真 min/max (RunningStats 内部本就维护)。
+ return NormStats(mean=self._mean, std=stddev, q01=q01, q99=q99, min=self._min, max=self._max)
def _adjust_histograms(self):
"""Adjust histograms when min or max changes."""
diff --git a/src/openpi/training/checkpoints.py b/src/openpi/training/checkpoints.py
index f32a831..fbdd572 100644
--- a/src/openpi/training/checkpoints.py
+++ b/src/openpi/training/checkpoints.py
@@ -41,8 +41,12 @@ def initialize_checkpoint_dir(
checkpoint_dir,
item_handlers={
"assets": CallbackHandler(),
- "train_state": ocp.PyTreeCheckpointHandler(),
- "params": ocp.PyTreeCheckpointHandler(),
+ # use_ocdbt=False: OCDBT 的 manifest 原子写语义在 NFS 类网络盘
+ # (平台 workspace PVC) 上会产生读回不一致 (DATA_LOSS: Length in
+ # header does not match); 关闭后走树状文件布局, 网络盘安全。
+ # 读取侧 orbax 自动识别格式, 官方 OCDBT 权重恢复不受影响。
+ "train_state": ocp.PyTreeCheckpointHandler(use_ocdbt=False),
+ "params": ocp.PyTreeCheckpointHandler(use_ocdbt=False),
},
options=ocp.CheckpointManagerOptions(
max_to_keep=1,
diff --git a/src/openpi/training/config.py b/src/openpi/training/config.py
index 4ca47e1..6b3e2d8 100644
--- a/src/openpi/training/config.py
+++ b/src/openpi/training/config.py
@@ -81,6 +81,14 @@ class DataConfig:
model_transforms: _transforms.Group = dataclasses.field(default_factory=_transforms.Group)
# If true, will use quantile normalization. Otherwise, normal z-score normalization will be used.
use_quantile_norm: bool = False
+ # DataPipeline 改动 (design_v2 D4): 契约归一化模式显式注入 (None -> 官方
+ # use_quantile_norm 行为): "min_max" / "mean_std" / "quantile"。
+ norm_mode: str | None = None
+
+ # DataPipeline 改动 (design_v2 多数据集聚合): 主 repo_id 之外的附加数据集
+ # repo_id 列表 (共父目录, 与主数据集共用同一套 transforms / norm_stats)。
+ # create_torch_dataset 会对主 + 附加逐一建库并 ConcatDataset 拼接。
+ extra_repo_ids: tuple[str, ...] = ()
# Names of keys that will be used by the data loader to generate the action sequence. The length of the
# sequence is defined by the `action_horizon` field in the model config. This should be adjusted if your
diff --git a/src/openpi/training/data_loader.py b/src/openpi/training/data_loader.py
index e2ee7dd..9f11269 100644
--- a/src/openpi/training/data_loader.py
+++ b/src/openpi/training/data_loader.py
@@ -1,4 +1,5 @@
from collections.abc import Iterator, Sequence
+import dataclasses
import logging
import multiprocessing
import os
@@ -7,7 +8,12 @@ from typing import Literal, Protocol, SupportsIndex, TypeVar
import jax
import jax.numpy as jnp
-import lerobot.common.datasets.lerobot_dataset as lerobot_dataset
+# DataPipeline 改动 (design_v2 D1): LeRobot v3 读取 (lerobot.datasets.* 命名空间,
+# 与 data2lerobot 转换器同大版本线 >=0.4.0); v2 安装时回退旧命名空间。
+try:
+ import lerobot.datasets.lerobot_dataset as lerobot_dataset
+except ImportError: # pragma: no cover - v2 fallback
+ import lerobot.common.datasets.lerobot_dataset as lerobot_dataset
import numpy as np
import torch
@@ -127,6 +133,28 @@ class FakeDataset(Dataset):
return self._num_samples
+def _create_single_lerobot_dataset(repo_id: str, data_config: _config.DataConfig, action_horizon: int):
+ """DataPipeline 改动 (多数据集聚合): 单 repo 的 LeRobot 数据集构建, 供主/附加
+ 数据集复用; delta_timestamps 的秒换算按各数据集自身 fps (fps 可不同)。"""
+ dataset_meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
+ dataset = lerobot_dataset.LeRobotDataset(
+ repo_id,
+ delta_timestamps={
+ key: [t / dataset_meta.fps for t in range(action_horizon)] for key in data_config.action_sequence_keys
+ },
+ )
+
+ if data_config.prompt_from_task:
+ # 改动点: lerobot >=0.4 的 meta.tasks 为 DataFrame(index=任务文本, 列含 task_index),
+ # PromptFromLeRobotTask 期望 dict[int, str] (0.1.0 语义), 此处归一化。
+ tasks = dataset_meta.tasks
+ if hasattr(tasks, "loc"): # pandas DataFrame
+ tasks = {int(row.task_index): idx for idx, row in tasks.iterrows()}
+ dataset = TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(tasks)])
+
+ return dataset
+
+
def create_torch_dataset(
data_config: _config.DataConfig, action_horizon: int, model_config: _model.BaseModelConfig
) -> Dataset:
@@ -137,16 +165,19 @@ def create_torch_dataset(
if repo_id == "fake":
return FakeDataset(model_config, num_samples=1024)
- dataset_meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
- dataset = lerobot_dataset.LeRobotDataset(
- data_config.repo_id,
- delta_timestamps={
- key: [t / dataset_meta.fps for t in range(action_horizon)] for key in data_config.action_sequence_keys
- },
- )
+ dataset = _create_single_lerobot_dataset(repo_id, data_config, action_horizon)
- if data_config.prompt_from_task:
- dataset = TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(dataset_meta.tasks)])
+ # DataPipeline 改动 (多数据集聚合): 附加数据集与主数据集共用同一 DataConfig 的
+ # transforms / norm_stats (repo_id 逐个替换), ConcatDataset 顺序拼接。
+ extra_repo_ids = list(getattr(data_config, "extra_repo_ids", ()) or ())
+ if extra_repo_ids:
+ parts = [dataset]
+ for extra_id in extra_repo_ids:
+ sub_config = dataclasses.replace(data_config, repo_id=extra_id, extra_repo_ids=())
+ parts.append(_create_single_lerobot_dataset(extra_id, sub_config, action_horizon))
+ dataset = torch.utils.data.ConcatDataset(parts)
+ logging.info(f"multi-dataset concat: {len(parts)} datasets "
+ f"({', '.join([repo_id, *extra_repo_ids])}), total {len(dataset)} frames")
return dataset
@@ -180,12 +211,18 @@ def transform_dataset(dataset: Dataset, data_config: _config.DataConfig, *, skip
)
norm_stats = data_config.norm_stats
+ # DataPipeline 改动 (D4): norm_mode 显式注入 (None -> 官方 use_quantile_norm 行为)。
return TransformedDataset(
dataset,
[
*data_config.repack_transforms.inputs,
*data_config.data_transforms.inputs,
- _transforms.Normalize(norm_stats, use_quantiles=data_config.use_quantile_norm),
+ _transforms.Normalize(
+ norm_stats,
+ use_quantiles=data_config.use_quantile_norm,
+ mode=data_config.norm_mode,
+ clip_to_quantiles=data_config.norm_mode == "mean_std",
+ ),
*data_config.model_transforms.inputs,
],
)
diff --git a/src/openpi/training/sharding.py b/src/openpi/training/sharding.py
index 344c550..ec4f382 100644
--- a/src/openpi/training/sharding.py
+++ b/src/openpi/training/sharding.py
@@ -7,13 +7,37 @@ import numpy as np
BATCH_AXIS = "batch"
FSDP_AXIS = "fsdp"
# In FSDP, we shard the data across both the batch and FSDP axes.
+# DataPipeline 注记: jax>=0.11 下当参数维恰可被 fsdp 整除时 (如 6 卡的 1152),
+# 数据复合轴与参数 fsdp 切分混合的 contraction 会报非法 sharding (旧版 jax 隐式
+# all-gather 不再自动插入); LoRA/低显存路径用 FSDP_DEVICES=1 (数据并行, 参数
+# replicate) 回避, 全量 FSDP 路径需显式 reshard 改造 (待 80GB 卡场景处理)
DATA_AXIS = (BATCH_AXIS, FSDP_AXIS)
+def data_axis_spec(mesh: jax.sharding.Mesh) -> tuple[str, ...]:
+ """DataPipeline 改动: 数据分片轴统一为纯 batch 轴。
+
+ - 单设备: 空 spec (replicated) — jax>=0.11 Explicit mesh 下任何轴名标记
+ (即使份额为 1) 都会触发词表 embedding gather 的输出分片歧义断言;
+ - 多设备: 仅 'batch' 轴 (数据并行度 = device_count / fsdp_devices)。
+ 官方复合轴 (batch, fsdp) 在 jax>=0.11 下与参数 fsdp 切分混合的
+ contraction 不再自动插入 all-gather (DenseGeneral 输出 fsdp 双重
+ 占用 → 非法 sharding), 故数据不再沾 fsdp 轴; fsdp 轴仅用于参数。
+ """
+ if mesh.devices.size == 1:
+ return ()
+ return (BATCH_AXIS,)
+
+
class _MeshState:
active_mesh: jax.sharding.Mesh | None = None
+def active_mesh() -> jax.sharding.Mesh | None:
+ """DataPipeline 改动: 模型层 (gemma 词表 gather 的显式 out_sharding) 需要只读访问当前 mesh。"""
+ return _MeshState.active_mesh
+
+
def make_mesh(num_fsdp_devices: int) -> jax.sharding.Mesh:
if jax.device_count() % num_fsdp_devices != 0:
raise ValueError(
@@ -38,11 +62,26 @@ def set_mesh(mesh: jax.sharding.Mesh):
def activation_sharding_constraint(pytree):
- if _MeshState.active_mesh is None:
+ # DataPipeline 改动: jax>=0.11 Explicit mesh 下 with_sharding_constraint 是
+ # 断言 (静态 spec 无法匹配 contraction 的自然分片); 改用 jax.sharding.reshard
+ # 强制层边界激活布局 = 维度 0 对齐 batch 轴、特征维 replicated (层间 all-gather,
+ # 即 FSDP 的标准层边界语义; MLP 中间大激活保持 fsdp 切分省显存)。
+ # 这同时满足 scan 对 carry 进出同型的要求。
+ if _MeshState.active_mesh is None or _MeshState.active_mesh.shape[FSDP_AXIS] == 1:
return pytree
- return jax.lax.with_sharding_constraint(
- pytree, jax.sharding.NamedSharding(_MeshState.active_mesh, jax.sharding.PartitionSpec(DATA_AXIS))
- )
+ mesh = _MeshState.active_mesh
+
+ def _reshard(a):
+ if not hasattr(a, "shape") or not hasattr(a, "dtype"):
+ return a
+ # 1 维数组 (如 per-token 的 ar_mask [N]) 无 batch 维, 标记 dim0 为 batch
+ # 会在 broadcast 后污染目标维并阻塞 reduce; 只处理 dim0 为 batch 的数组
+ if len(a.shape) < 2:
+ return a
+ spec = jax.sharding.PartitionSpec(*((BATCH_AXIS,) + (None,) * (len(a.shape) - 1)))
+ return jax.sharding.reshard(a, jax.sharding.NamedSharding(mesh, spec))
+
+ return jax.tree_util.tree_map(_reshard, pytree)
def fsdp_sharding(
@@ -80,15 +119,35 @@ def fsdp_sharding(
if (arr_size := np.prod(array.shape) * np.dtype(array.dtype).itemsize) < min_size_bytes:
return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
- # shard matrices and larger tensors along the largest axis that is divisible by the fsdp dimension
- axes = np.argsort(array.shape)[::-1]
- spec = [None] * len(axes)
- for i in axes:
+ # DataPipeline 改动: attention 类参数一律 replicate。scan 将 27/18 层堆叠
+ # 为单个大权重, 阈值无法按层粒度放行; 而 attention 权重任何维度的切分
+ # 都会进入收缩维 (qkv 的 head_dim 在 MHA einsum 收缩; out proj 的输入
+ # 维在 attn_vec_einsum 收缩), jax>=0.11 下双边 fsdp 收缩必须显式
+ # out_sharding (flax MHA 内部不可注入), 故整体 replicate (约 1GB,
+ # 相对总参数是小头)。
+ _kp = jax.tree_util.keystr(kp)
+ if (
+ "MultiHeadDotProductAttention" in _kp
+ or "['attn']" in _kp
+ or "attn_vec_einsum" in _kp
+ or "q_einsum" in _kp
+ or "kv_einsum" in _kp
+ ):
+ return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
+
+ # DataPipeline 改动: 轴选择从"最大可整除轴优先"改为"最后维优先"。
+ # flax Dense kernel 为 [in, out], 收缩维是 dim0; 切最后维 (out 维) 时
+ # contraction 输出可自然分片, 不产生双边 fsdp 收缩歧义 (jax>=0.11 不再
+ # 自动插入 all-gather)。词表 embedding [vocab, d] 切 d 维后, gemma.encode
+ # 的 gather 以显式 out_sharding (d 维 replicated) 收敛, 即 FSDP 的
+ # 用时 all-gather 语义。
+ for i in reversed(range(len(array.shape))):
if array.shape[i] % mesh.shape[FSDP_AXIS] == 0:
if log:
logging.info(
f"Sharding {jax.tree_util.keystr(kp)} of shape {array.shape} ({arr_size / 2**20:.2f} MiB) along axis {i}"
)
+ spec = [None] * len(array.shape)
spec[i] = FSDP_AXIS
return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(*spec))
diff --git a/src/openpi/training/utils.py b/src/openpi/training/utils.py
index fe7f94d..bc7d155 100644
--- a/src/openpi/training/utils.py
+++ b/src/openpi/training/utils.py
@@ -13,6 +13,12 @@ from openpi.shared import array_typing as at
@at.typecheck
@struct.dataclass
class TrainState:
+ # 来源: openpi仓库 src/openpi/training/utils.py @ 15a9616
+ # 改动点: 1) 设 __no_type_check__ 跳过 jaxtyped 对 dataclass __init__ 的运行时检查 —
+ # jaxtyping>=0.3 移除了 _check_dataclass_annotations 豁免挂点 (issue #277 上游重构),
+ # eval_shape/tracing 下 step 会被以 ShapeDtypeStruct / NamedSharding 等抽象占位
+ # 初始化, beartype 误拒; __no_type_check__ 是 jaxtyped 对 dataclass 的官方跳过开关
+ __no_type_check__ = True
step: at.Int[at.ArrayLike, ""]
params: nnx.State
model_def: nnx.GraphDef[_model.BaseModel]
diff --git a/src/openpi/transforms.py b/src/openpi/transforms.py
index 272375e..8764739 100644
--- a/src/openpi/transforms.py
+++ b/src/openpi/transforms.py
@@ -118,25 +118,43 @@ class Normalize(DataTransformFn):
use_quantiles: bool = False
# If true, will raise an error if any of the keys in the norm stats are not present in the data.
strict: bool = False
+ # DataPipeline 改动 (design_v2 D4/D19): 归一化模式显式化。
+ # None -> 由 use_quantiles 推断 (官方行为: quantile / mean_std);
+ # "min_max": 有界 -> [-1,1] (min_max 统计, 兼容 discrete_state_input, 契约默认);
+ # "quantile" / "mean_std": 与官方两档一致。
+ mode: str | None = None
+ # mean_std + discrete_state 注入时 clip 到 q01/q99 的归一化像 (D19, 对齐作者
+ # quantile 有界化意图), 避免 256-bin 离散化截断。
+ clip_to_quantiles: bool = False
def __post_init__(self):
- if self.norm_stats is not None and self.use_quantiles:
+ resolved_mode = self.mode or ("quantile" if self.use_quantiles else "mean_std")
+ if resolved_mode not in ("mean_std", "quantile", "min_max"):
+ raise ValueError(f"unsupported normalize mode: {resolved_mode}")
+ if self.norm_stats is not None and resolved_mode == "quantile":
_assert_quantile_stats(self.norm_stats)
+ object.__setattr__(self, "mode", resolved_mode)
def __call__(self, data: DataDict) -> DataDict:
if self.norm_stats is None:
return data
- return apply_tree(
- data,
- self.norm_stats,
- self._normalize_quantile if self.use_quantiles else self._normalize,
- strict=self.strict,
- )
+ fn = {
+ "mean_std": self._normalize,
+ "quantile": self._normalize_quantile,
+ "min_max": self._normalize_min_max,
+ }[self.mode]
+ return apply_tree(data, self.norm_stats, fn, strict=self.strict)
def _normalize(self, x, stats: NormStats):
mean, std = stats.mean[..., : x.shape[-1]], stats.std[..., : x.shape[-1]]
- return (x - mean) / (std + 1e-6)
+ out = (x - mean) / (std + 1e-6)
+ if self.clip_to_quantiles and stats.q01 is not None and stats.q99 is not None:
+ q01, q99 = stats.q01[..., : x.shape[-1]], stats.q99[..., : x.shape[-1]]
+ lo = (q01 - mean) / (std + 1e-6)
+ hi = (q99 - mean) / (std + 1e-6)
+ out = np.clip(out, lo, hi)
+ return out
def _normalize_quantile(self, x, stats: NormStats):
assert stats.q01 is not None
@@ -144,28 +162,44 @@ class Normalize(DataTransformFn):
q01, q99 = stats.q01[..., : x.shape[-1]], stats.q99[..., : x.shape[-1]]
return (x - q01) / (q99 - q01 + 1e-6) * 2.0 - 1.0
+ def _normalize_min_max(self, x, stats: NormStats):
+ lo = stats.min if stats.min is not None else stats.q01
+ hi = stats.max if stats.max is not None else stats.q99
+ assert lo is not None and hi is not None, (
+ "min_max mode requires min/max stats (recompute norm_stats); "
+ "legacy stats fallback to q01/q99"
+ )
+ lo, hi = lo[..., : x.shape[-1]], hi[..., : x.shape[-1]]
+ return (x - lo) / (hi - lo + 1e-6) * 2.0 - 1.0
+
@dataclasses.dataclass(frozen=True)
class Unnormalize(DataTransformFn):
norm_stats: at.PyTree[NormStats] | None
# If true, will use quantile normalization. Otherwise, normal z-score normalization will be used.
use_quantiles: bool = False
+ # DataPipeline 改动 (D4): 与 Normalize.mode 对称。
+ mode: str | None = None
def __post_init__(self):
- if self.norm_stats is not None and self.use_quantiles:
+ resolved_mode = self.mode or ("quantile" if self.use_quantiles else "mean_std")
+ if resolved_mode not in ("mean_std", "quantile", "min_max"):
+ raise ValueError(f"unsupported unnormalize mode: {resolved_mode}")
+ if self.norm_stats is not None and resolved_mode == "quantile":
_assert_quantile_stats(self.norm_stats)
+ object.__setattr__(self, "mode", resolved_mode)
def __call__(self, data: DataDict) -> DataDict:
if self.norm_stats is None:
return data
+ fn = {
+ "mean_std": self._unnormalize,
+ "quantile": self._unnormalize_quantile,
+ "min_max": self._unnormalize_min_max,
+ }[self.mode]
# Make sure that all the keys in the norm stats are present in the data.
- return apply_tree(
- data,
- self.norm_stats,
- self._unnormalize_quantile if self.use_quantiles else self._unnormalize,
- strict=True,
- )
+ return apply_tree(data, self.norm_stats, fn, strict=True)
def _unnormalize(self, x, stats: NormStats):
mean = pad_to_dim(stats.mean, x.shape[-1], axis=-1, value=0.0)
@@ -180,6 +214,16 @@ class Unnormalize(DataTransformFn):
return np.concatenate([(x[..., :dim] + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01, x[..., dim:]], axis=-1)
return (x + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01
+ def _unnormalize_min_max(self, x, stats: NormStats):
+ lo = stats.min if stats.min is not None else stats.q01
+ hi = stats.max if stats.max is not None else stats.q99
+ assert lo is not None and hi is not None
+ if (dim := lo.shape[-1]) < x.shape[-1]:
+ return np.concatenate(
+ [(x[..., :dim] + 1.0) / 2.0 * (hi - lo + 1e-6) + lo, x[..., dim:]], axis=-1
+ )
+ return (x + 1.0) / 2.0 * (hi - lo + 1e-6) + lo
+
@dataclasses.dataclass(frozen=True)
class ResizeImages(DataTransformFn):
@@ -324,6 +368,44 @@ class PromptFromLeRobotTask(DataTransformFn):
return {**data, "prompt": prompt}
+@dataclasses.dataclass(frozen=True)
+class ImagesToHWC(DataTransformFn):
+ """DataPipeline 改动 (design_v2 D1): 对齐官方 libero_policy._parse_image 语义 —
+ lerobot >=0.4 视频帧为 float32 [0,1] CHW, 官方数据管线为 uint8 HWC
+ (ResizeImages 走 PIL uint8 golden; Observation.from_dict 再转 [-1,1] float,
+ torch 分支并 permute CHW)。float 输入 clip [0,1] 后 *255 取整, 对
+ uint8/HWC 输入幂等无操作。
+ """
+
+ def __call__(self, data: DataDict) -> DataDict:
+ images = data.get("image")
+ if not isinstance(images, dict):
+ return data
+
+ def _parse(img):
+ img = np.asarray(img)
+ if np.issubdtype(img.dtype, np.floating):
+ img = (255 * np.clip(img, 0.0, 1.0)).astype(np.uint8)
+ if img.ndim == 3 and img.shape[0] <= 4 < img.shape[-1]:
+ img = np.transpose(img, (1, 2, 0))
+ return img
+
+ return {**data, "image": {key: _parse(value) for key, value in images.items()}}
+
+
+@dataclasses.dataclass(frozen=True)
+class BuildImageMasks(DataTransformFn):
+ """DataPipeline 改动 (design_v2 D15): 为契约 N 槽相机生成 image_mask (每槽 True)。
+
+ 槽位完备性由契约 key_map 校验保证 (每个声明相机恰好被映射一次), 无需官方
+ aloha_policy 式缺路填黑 + False 分支。缺此 transform 时 Observation.from_dict
+ 取 data["image_mask"] 抛 KeyError。
+ """
+
+ def __call__(self, data: DataDict) -> DataDict:
+ return {**data, "image_mask": {key: np.True_ for key in data["image"]}}
+
+
@dataclasses.dataclass(frozen=True)
class PadStatesAndActions(DataTransformFn):
"""Zero-pads states and actions to the model action dimension."""
--
2.34.1