926 lines
46 KiB
Diff
926 lines
46 KiB
Diff
From da7ce8b6720afdbd28054a3ca9d804eb60c9f9f8 Mon Sep 17 00:00:00 2001
|
|
From: "chao01.wu" <chao01.wu@local>
|
|
Date: Tue, 8 Sep 2026 21:28:01 +0800
|
|
Subject: [PATCH] DataPipeline: 1:1-only revert (aspect/pos_emb slice removed,
|
|
siglip+modeling_siglip restored to official) + multi-dataset concat
|
|
(DataConfig.extra_repo_ids, create_torch_dataset Concat)
|
|
|
|
---
|
|
examples/convert_jax_model_to_pytorch.py | 6 +-
|
|
scripts/train_pytorch.py | 19 ++-
|
|
src/openpi/models/gemma.py | 13 +-
|
|
src/openpi/models/model.py | 40 ++++++-
|
|
src/openpi/models/pi0.py | 40 ++++++-
|
|
src/openpi/models/pi0_config.py | 25 ++--
|
|
src/openpi/models_pytorch/pi0_pytorch.py | 10 +-
|
|
.../models_pytorch/preprocessing_pytorch.py | 5 +
|
|
src/openpi/shared/array_typing.py | 22 ++--
|
|
src/openpi/shared/image_tools.py | 10 +-
|
|
src/openpi/shared/nnx_utils.py | 8 +-
|
|
src/openpi/shared/normalize.py | 7 +-
|
|
src/openpi/training/checkpoints.py | 8 +-
|
|
src/openpi/training/config.py | 8 ++
|
|
src/openpi/training/data_loader.py | 59 +++++++--
|
|
src/openpi/training/sharding.py | 75 ++++++++++--
|
|
src/openpi/training/utils.py | 6 +
|
|
src/openpi/transforms.py | 112 +++++++++++++++---
|
|
18 files changed, 402 insertions(+), 71 deletions(-)
|
|
|
|
diff --git a/examples/convert_jax_model_to_pytorch.py b/examples/convert_jax_model_to_pytorch.py
|
|
index 632c0b8..e4459bb 100644
|
|
--- a/examples/convert_jax_model_to_pytorch.py
|
|
+++ b/examples/convert_jax_model_to_pytorch.py
|
|
@@ -290,7 +290,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
|
|
llm_mlp_linear = state_dict.pop(f"llm/layers/mlp_{num_expert}/linear{suffix}")
|
|
|
|
# Check if we have Dense layers (for pi05/adaptive normalization) or scale layers (for regular pi0)
|
|
- if "pi05" in checkpoint_dir:
|
|
+ if pi05:
|
|
# Pi05 with adaptive normalization
|
|
llm_input_layernorm_bias = state_dict.pop(f"llm/layers/pre_attention_norm_{num_expert}/Dense_0/bias{suffix}")
|
|
llm_post_attention_layernorm_bias = state_dict.pop(f"llm/layers/pre_ffw_norm_{num_expert}/Dense_0/bias{suffix}")
|
|
@@ -345,7 +345,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
|
|
i
|
|
].transpose()
|
|
|
|
- if "pi05" in checkpoint_dir:
|
|
+ if pi05:
|
|
# Pi05 with adaptive normalization - use Dense layer parameters directly
|
|
state_dict[f"paligemma_with_expert.gemma_expert.model.layers.{i}.input_layernorm.dense.bias"] = (
|
|
llm_input_layernorm_bias[i]
|
|
@@ -369,7 +369,7 @@ def slice_gemma_state_dict(state_dict, config, *, num_expert, checkpoint_dir, pi
|
|
)
|
|
|
|
# Handle final norm layer
|
|
- if "pi05" in checkpoint_dir:
|
|
+ if pi05:
|
|
# Pi05 with adaptive normalization - use Dense layer parameters directly
|
|
final_norm_bias = state_dict.pop(f"llm/final_norm_{num_expert}/Dense_0/bias{suffix}")
|
|
final_norm_kernel = state_dict.pop(f"llm/final_norm_{num_expert}/Dense_0/kernel{suffix}")
|
|
diff --git a/scripts/train_pytorch.py b/scripts/train_pytorch.py
|
|
index c7ddd2b..fa698ca 100644
|
|
--- a/scripts/train_pytorch.py
|
|
+++ b/scripts/train_pytorch.py
|
|
@@ -455,8 +455,25 @@ def train_loop(config: _config.TrainConfig):
|
|
end_lr = config.lr_schedule.decay_lr
|
|
|
|
# Create optimizer with config parameters
|
|
+ # DataPipeline 改动 (design_v2 §5.3 注记): FREEZE_BACKBONE=1 冻结 Paligemma 主干
|
|
+ # (vision/llm/expert), 只训练 action_in_proj/action_out_proj/time_mlp 头 —
|
|
+ # 32GB 卡全参 AdamW (params+grads+opt 状态 ~36GB fp32) 不可行时的低显存路径
|
|
+ # (官方 torch 栈无 LoRA 支持, 冒烟/验收语义为可训练可转化, 不要求全参收敛)
|
|
+ trainable_params = model.parameters()
|
|
+ if os.environ.get("FREEZE_BACKBONE", "").strip().lower() in ("1", "true", "yes"):
|
|
+ frozen, trainable = 0, 0
|
|
+ for name, p in model.named_parameters():
|
|
+ if any(head in name for head in ("action_in_proj", "action_out_proj", "time_mlp")):
|
|
+ p.requires_grad = True
|
|
+ trainable += p.numel()
|
|
+ else:
|
|
+ p.requires_grad = False
|
|
+ frozen += p.numel()
|
|
+ trainable_params = [p for p in model.parameters() if p.requires_grad]
|
|
+ logging.info(f"[freeze] trainable={trainable/1e6:.1f}M frozen={frozen/1e6:.1f}M params")
|
|
+
|
|
optim = torch.optim.AdamW(
|
|
- model.parameters(),
|
|
+ trainable_params,
|
|
lr=peak_lr,
|
|
betas=(config.optimizer.b1, config.optimizer.b2),
|
|
eps=config.optimizer.eps,
|
|
diff --git a/src/openpi/models/gemma.py b/src/openpi/models/gemma.py
|
|
index d1623d1..7fbb113 100644
|
|
--- a/src/openpi/models/gemma.py
|
|
+++ b/src/openpi/models/gemma.py
|
|
@@ -146,7 +146,18 @@ class Embedder(nn.Module):
|
|
)
|
|
|
|
def encode(self, x):
|
|
- x = self.input_embedding_table[(x,)]
|
|
+ # DataPipeline 改动: jax>=0.11 下词表 gather (replicated embedding 表 x
|
|
+ # batch 分片 indices) 无法自动解析输出 sharding (模型内 prompt 与 state
|
|
+ # 离散 token concat 后继承 batch 分片)。按 jax 官方姿势显式声明输出
|
|
+ # 分片 = indices 各维 + embedding 维, 维度 0 对齐 batch 轴。
|
|
+ mesh = sharding.active_mesh()
|
|
+ if mesh is not None and mesh.devices.size > 1:
|
|
+ spec = jax.sharding.PartitionSpec(sharding.BATCH_AXIS, *((None,) * (x.ndim - 1)))
|
|
+ x = self.input_embedding_table.at[x].get(
|
|
+ out_sharding=jax.sharding.NamedSharding(mesh, spec)
|
|
+ )
|
|
+ else:
|
|
+ x = self.input_embedding_table[(x,)]
|
|
x *= jnp.sqrt(self.embed_dim).astype(x.dtype)
|
|
return x
|
|
|
|
diff --git a/src/openpi/models/model.py b/src/openpi/models/model.py
|
|
index 29618b4..073e802 100644
|
|
--- a/src/openpi/models/model.py
|
|
+++ b/src/openpi/models/model.py
|
|
@@ -3,6 +3,7 @@ from collections.abc import Sequence
|
|
import dataclasses
|
|
import enum
|
|
import logging
|
|
+import os
|
|
import pathlib
|
|
from typing import Generic, TypeVar
|
|
|
|
@@ -81,6 +82,10 @@ IMAGE_RESOLUTION = (224, 224)
|
|
@at.typecheck
|
|
@struct.dataclass
|
|
class Observation(Generic[ArrayT]):
|
|
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
|
|
+ # 改动点: 1) 同 TrainState — __no_type_check__ 跳过 jaxtyped dataclass __init__
|
|
+ # 检查 (jaxtyping>=0.11 tracing 下 images dict 以 None 占位初始化被误拒)
|
|
+ __no_type_check__ = True
|
|
"""Holds observations, i.e., inputs to the model.
|
|
|
|
See `Observation.from_dict` to see the expected dictionary form. This is the format
|
|
@@ -141,6 +146,20 @@ class Observation(Generic[ArrayT]):
|
|
Actions = at.Float[ArrayT, "*b ah ad"]
|
|
|
|
|
|
+def _reshard_to_data_axis(x):
|
|
+ """DataPipeline 改动: jax>=0.11 vmap/concat 的 sharding 一致性辅助。
|
|
+
|
|
+ 多进程 mesh 下把数组 reshard 到数据轴 (与 training.sharding.data_axis_spec
|
|
+ 同语义: fsdp=1 时退化单 batch 轴); 无 mesh 上下文 (单进程推理/转换自检) 时
|
|
+ 原样返回。
|
|
+ """
|
|
+ mesh = jax.sharding.get_abstract_mesh()
|
|
+ if not mesh.empty and mesh.size > 1:
|
|
+ axis = ("batch",) if mesh.shape["fsdp"] == 1 else ("batch", "fsdp")
|
|
+ x = jax.sharding.reshard(x, jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(axis)))
|
|
+ return x
|
|
+
|
|
+
|
|
def preprocess_observation(
|
|
rng: at.KeyArrayLike | None,
|
|
observation: Observation,
|
|
@@ -165,7 +184,11 @@ def preprocess_observation(
|
|
logger.info(f"Resizing image {key} from {image.shape[1:3]} to {image_resolution}")
|
|
image = image_tools.resize_with_pad(image, *image_resolution)
|
|
|
|
- if train:
|
|
+ if train and os.environ.get("OPENPI_DISABLE_AUGMENT", "").strip().lower() not in ("1", "true", "yes"):
|
|
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
|
|
+ # 改动点: 1) OPENPI_DISABLE_AUGMENT=1 跳过 augmax 增广 — augmax (jax 0.4
|
|
+ # 时代库) 与 jax>=0.11 不兼容 (ColorJitter pixelwise 内部崩, 风险项
|
|
+ # R-13), 冒烟/推理不受影响; 正式训练前需替换或修复增广实现
|
|
# Convert from [-1, 1] to [0, 1] for augmax.
|
|
image = image / 2.0 + 0.5
|
|
|
|
@@ -181,6 +204,12 @@ def preprocess_observation(
|
|
augmax.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5),
|
|
]
|
|
sub_rngs = jax.random.split(rng, image.shape[0])
|
|
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
|
|
+ # 改动点: 1) jax>=0.11 要求 vmap 消去维度的 sharding 一致且全 Explicit mesh
|
|
+ # 下 with_sharding_constraint 是 assert — 用 reshard 把 split 产生的
|
|
+ # replicated sub_rngs 真分片到数据轴 (与 image 一致); 无 mesh 上下文
|
|
+ # (单进程推理/转换自检) 时跳过
|
|
+ sub_rngs = _reshard_to_data_axis(sub_rngs)
|
|
image = jax.vmap(augmax.Chain(*transforms))(sub_rngs, image)
|
|
|
|
# Back to [-1, 1].
|
|
@@ -312,7 +341,14 @@ def restore_params(
|
|
|
|
with ocp.PyTreeCheckpointer() as ckptr:
|
|
metadata = ckptr.metadata(params_path)
|
|
- item = {"params": metadata["params"]}
|
|
+ # 来源: openpi仓库 src/openpi/models/model.py @ 15a9616
|
|
+ # 改动点: 1) orbax>=0.12 的 metadata() 返回 StepMetadata (树结构在
|
|
+ # item_metadata["params"]), 旧版直接下标; 双形态兼容
|
|
+ if hasattr(metadata, "item_metadata"):
|
|
+ params_metadata = metadata.item_metadata["params"]
|
|
+ else:
|
|
+ params_metadata = metadata["params"]
|
|
+ item = {"params": params_metadata}
|
|
|
|
params = ckptr.restore(
|
|
params_path,
|
|
diff --git a/src/openpi/models/pi0.py b/src/openpi/models/pi0.py
|
|
index ae7c459..09dddda 100644
|
|
--- a/src/openpi/models/pi0.py
|
|
+++ b/src/openpi/models/pi0.py
|
|
@@ -11,6 +11,7 @@ from openpi.models import model as _model
|
|
from openpi.models import pi0_config
|
|
import openpi.models.gemma as _gemma
|
|
import openpi.models.siglip as _siglip
|
|
+import openpi.training.sharding as _sharding
|
|
from openpi.shared import array_typing as at
|
|
|
|
logger = logging.getLogger("openpi")
|
|
@@ -67,6 +68,9 @@ class Pi0(_model.BaseModel):
|
|
def __init__(self, config: pi0_config.Pi0Config, rngs: nnx.Rngs):
|
|
super().__init__(config.action_dim, config.action_horizon, config.max_token_len)
|
|
self.pi05 = config.pi05
|
|
+ # DataPipeline 改动 (D15/D2): 保存 config 引用, preprocess_observation 与
|
|
+ # SigLIP 构造从契约派生字段取图像槽位/分辨率/pos_emb 行切片。
|
|
+ self.config = config
|
|
paligemma_config = _gemma.get_config(config.paligemma_variant)
|
|
action_expert_config = _gemma.get_config(config.action_expert_variant)
|
|
# TODO: rewrite gemma in NNX. For now, use bridge.
|
|
@@ -177,7 +181,12 @@ class Pi0(_model.BaseModel):
|
|
action_expert_tokens = action_time_tokens
|
|
adarms_cond = None
|
|
tokens.append(action_expert_tokens)
|
|
- input_mask.append(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
|
|
+ # 来源: openpi仓库 src/openpi/models/pi0.py @ 15a9616
|
|
+ # 改动点: 1) jax>=0.11 要求 concat 操作数 sharding 一致 — jit 内 jnp.ones 生成的
|
|
+ # suffix_mask 是 replicated, 与来自数据分片的 prefix_mask 冲突; 约束到数据轴
|
|
+ # (无 mesh 上下文时跳过)
|
|
+ suffix_mask = _model._reshard_to_data_axis(jnp.ones(action_expert_tokens.shape[:2], dtype=jnp.bool_))
|
|
+ input_mask.append(suffix_mask)
|
|
# image/language/state inputs do not attend to action tokens
|
|
ar_mask += [True] + ([False] * (self.action_horizon - 1))
|
|
tokens = jnp.concatenate(tokens, axis=1)
|
|
@@ -190,9 +199,20 @@ class Pi0(_model.BaseModel):
|
|
self, rng: at.KeyArrayLike, observation: _model.Observation, actions: _model.Actions, *, train: bool = False
|
|
) -> at.Float[at.Array, "*b ah"]:
|
|
preprocess_rng, noise_rng, time_rng = jax.random.split(rng, 3)
|
|
- observation = _model.preprocess_observation(preprocess_rng, observation, train=train)
|
|
+ # DataPipeline 改动 (D15): 图像槽位/分辨率取契约派生字段 (默认官方值)。
|
|
+ observation = _model.preprocess_observation(
|
|
+ preprocess_rng,
|
|
+ observation,
|
|
+ train=train,
|
|
+ image_keys=self.config.resolved_image_keys,
|
|
+ )
|
|
|
|
batch_shape = actions.shape[:-2]
|
|
+ # DataPipeline 改动: jax>=0.11 下不同子路径 (图像/语言 vs actions/time)
|
|
+ # 的自然分片可能物理等价但 spec 不同 (batch 轴份额 1 时 P('batch') 与
|
|
+ # P(('batch','fsdp')) 并存), concatenate 严格比较即报错; 入口统一
|
|
+ # reshard 到维度 0 对齐 batch 轴
|
|
+ observation = _sharding.activation_sharding_constraint(observation)
|
|
noise = jax.random.normal(noise_rng, actions.shape)
|
|
time = jax.random.beta(time_rng, 1.5, 1, batch_shape) * 0.999 + 0.001
|
|
time_expanded = time[..., None, None]
|
|
@@ -202,6 +222,14 @@ class Pi0(_model.BaseModel):
|
|
# one big forward pass of prefix + suffix at once
|
|
prefix_tokens, prefix_mask, prefix_ar_mask = self.embed_prefix(observation)
|
|
suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = self.embed_suffix(observation, x_t, time)
|
|
+ # DataPipeline 改动: prefix (observation 路径) 与 suffix (actions/time 路径)
|
|
+ # 的自然分片在 batch 轴份额 1 时会得到 P('batch') 与 P(('batch','fsdp'))
|
|
+ # 两种物理等价但 spec 不同的布局, concatenate 严格比较报错; 统一 reshard
|
|
+ _unify = _sharding.activation_sharding_constraint
|
|
+ prefix_tokens, prefix_mask, prefix_ar_mask = _unify((prefix_tokens, prefix_mask, prefix_ar_mask))
|
|
+ suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond = _unify(
|
|
+ (suffix_tokens, suffix_mask, suffix_ar_mask, adarms_cond)
|
|
+ )
|
|
input_mask = jnp.concatenate([prefix_mask, suffix_mask], axis=1)
|
|
ar_mask = jnp.concatenate([prefix_ar_mask, suffix_ar_mask], axis=0)
|
|
attn_mask = make_attn_mask(input_mask, ar_mask)
|
|
@@ -222,7 +250,13 @@ class Pi0(_model.BaseModel):
|
|
num_steps: int | at.Int[at.Array, ""] = 10,
|
|
noise: at.Float[at.Array, "b ah ad"] | None = None,
|
|
) -> _model.Actions:
|
|
- observation = _model.preprocess_observation(None, observation, train=False)
|
|
+ # DataPipeline 改动 (D15): 同 compute_loss, 契约派生槽位/分辨率。
|
|
+ observation = _model.preprocess_observation(
|
|
+ None,
|
|
+ observation,
|
|
+ train=False,
|
|
+ image_keys=self.config.resolved_image_keys,
|
|
+ )
|
|
# note that we use the convention more common in diffusion literature, where t=1 is noise and t=0 is the target
|
|
# distribution. yes, this is the opposite of the pi0 paper, and I'm sorry.
|
|
dt = -1.0 / num_steps
|
|
diff --git a/src/openpi/models/pi0_config.py b/src/openpi/models/pi0_config.py
|
|
index 584d83f..078c628 100644
|
|
--- a/src/openpi/models/pi0_config.py
|
|
+++ b/src/openpi/models/pi0_config.py
|
|
@@ -32,6 +32,11 @@ class Pi0Config(_model.BaseModelConfig):
|
|
# This config option is not used directly by the model, but it is read by the ModelTransformFactory.
|
|
discrete_state_input: bool = None # type: ignore
|
|
|
|
+ # DataPipeline 改动 (design_v2 D15, 契约驱动): 相机槽位。
|
|
+ # 默认 None = 官方行为 (model.IMAGE_KEYS 3 槽, 224x224, 完整 16x16 网格), 向后兼容。
|
|
+ # 契约注入示例: image_keys=("cam1","cam2","cam3","cam4")
|
|
+ image_keys: tuple[str, ...] | None = None
|
|
+
|
|
pytorch_compile_mode: str | None = "max-autotune"
|
|
|
|
def __post_init__(self):
|
|
@@ -47,6 +52,11 @@ class Pi0Config(_model.BaseModelConfig):
|
|
"max-autotune-no-cudagraphs",
|
|
]
|
|
|
|
+ @property
|
|
+ def resolved_image_keys(self) -> tuple[str, ...]:
|
|
+ """契约槽位或官方默认 (cam 序号即 prefix token 拼接顺序, D15)。"""
|
|
+ return self.image_keys if self.image_keys else _model.IMAGE_KEYS
|
|
+
|
|
@property
|
|
@override
|
|
def model_type(self) -> _model.ModelType:
|
|
@@ -62,21 +72,16 @@ class Pi0Config(_model.BaseModelConfig):
|
|
|
|
@override
|
|
def inputs_spec(self, *, batch_size: int = 1) -> tuple[_model.Observation, _model.Actions]:
|
|
+ # DataPipeline 改动 (D15): 图像槽位由契约派生 (resolved_image_keys),
|
|
+ # 默认与官方一致 (3 槽 224x224)。
|
|
image_spec = jax.ShapeDtypeStruct([batch_size, *_model.IMAGE_RESOLUTION, 3], jnp.float32)
|
|
image_mask_spec = jax.ShapeDtypeStruct([batch_size], jnp.bool_)
|
|
|
|
+ keys = self.resolved_image_keys
|
|
with at.disable_typechecking():
|
|
observation_spec = _model.Observation(
|
|
- images={
|
|
- "base_0_rgb": image_spec,
|
|
- "left_wrist_0_rgb": image_spec,
|
|
- "right_wrist_0_rgb": image_spec,
|
|
- },
|
|
- image_masks={
|
|
- "base_0_rgb": image_mask_spec,
|
|
- "left_wrist_0_rgb": image_mask_spec,
|
|
- "right_wrist_0_rgb": image_mask_spec,
|
|
- },
|
|
+ images={key: image_spec for key in keys},
|
|
+ image_masks={key: image_mask_spec for key in keys},
|
|
state=jax.ShapeDtypeStruct([batch_size, self.action_dim], jnp.float32),
|
|
tokenized_prompt=jax.ShapeDtypeStruct([batch_size, self.max_token_len], jnp.int32),
|
|
tokenized_prompt_mask=jax.ShapeDtypeStruct([batch_size, self.max_token_len], bool),
|
|
diff --git a/src/openpi/models_pytorch/pi0_pytorch.py b/src/openpi/models_pytorch/pi0_pytorch.py
|
|
index e68ddb7..f8dee9d 100644
|
|
--- a/src/openpi/models_pytorch/pi0_pytorch.py
|
|
+++ b/src/openpi/models_pytorch/pi0_pytorch.py
|
|
@@ -160,8 +160,14 @@ class PI0Pytorch(nn.Module):
|
|
return torch.where(att_2d_masks_4d, 0.0, -2.3819763e38)
|
|
|
|
def _preprocess_observation(self, observation, *, train=True):
|
|
- """Helper method to preprocess observation."""
|
|
- observation = _preprocessing.preprocess_observation_pytorch(observation, train=train)
|
|
+ """Helper model to preprocess observation."""
|
|
+ # DataPipeline 改动 (design_v2 D15): 图像槽位取契约派生字段
|
|
+ # (Pi0Config.resolved_image_keys, 默认官方值), 与 JAX 侧一致。
|
|
+ observation = _preprocessing.preprocess_observation_pytorch(
|
|
+ observation,
|
|
+ train=train,
|
|
+ image_keys=self.config.resolved_image_keys,
|
|
+ )
|
|
return (
|
|
list(observation.images.values()),
|
|
list(observation.image_masks.values()),
|
|
diff --git a/src/openpi/models_pytorch/preprocessing_pytorch.py b/src/openpi/models_pytorch/preprocessing_pytorch.py
|
|
index 33c94a5..d53cbc7 100644
|
|
--- a/src/openpi/models_pytorch/preprocessing_pytorch.py
|
|
+++ b/src/openpi/models_pytorch/preprocessing_pytorch.py
|
|
@@ -39,6 +39,11 @@ def preprocess_observation_pytorch(
|
|
|
|
# TODO: This is a hack to handle both [B, C, H, W] and [B, H, W, C] formats
|
|
# Handle both [B, C, H, W] and [B, H, W, C] formats
|
|
+ # DataPipeline 改动: HWC 输入 (B,H,W,C: 尾维==3 且 shape[1]!=3) 先归一到
|
|
+ # CHW — 官方 torch 数据管线为 CHW, 但 LeRobot 侧共用 transform 链为 HWC;
|
|
+ # 下游 embed_image -> transformers get_image_features 只接受 CHW
|
|
+ if image.shape[-1] == 3 and image.shape[1] != 3:
|
|
+ image = image.permute(0, 3, 1, 2) # [B, H, W, C] -> [B, C, H, W]
|
|
is_channels_first = image.shape[1] == 3 # Check if channels are in dimension 1
|
|
|
|
if is_channels_first:
|
|
diff --git a/src/openpi/shared/array_typing.py b/src/openpi/shared/array_typing.py
|
|
index 569eafe..0be9e74 100644
|
|
--- a/src/openpi/shared/array_typing.py
|
|
+++ b/src/openpi/shared/array_typing.py
|
|
@@ -26,21 +26,25 @@ import torch
|
|
# the problem is that custom PyTree nodes are sometimes initialized with arbitrary types (e.g., `jax.ShapeDtypeStruct`,
|
|
# `jax.Sharding`, or even <object>) due to JAX tracing operations. this patch skips typechecking when the stack trace
|
|
# contains `jax._src.tree_util`, which should only be the case during tree unflattening.
|
|
-_original_check_dataclass_annotations = jaxtyping._decorator._check_dataclass_annotations # noqa: SLF001
|
|
+# 来源: openpi仓库 src/openpi/shared/array_typing.py @ 15a9616
|
|
+# 改动点: 1) jaxtyping>=0.3 移除了 _check_dataclass_annotations 私有 API (issue #277 已上游修复),
|
|
+# 改为存在性判断: 旧版打 patch, 新版跳过
|
|
+_original_check_dataclass_annotations = getattr(jaxtyping._decorator, "_check_dataclass_annotations", None) # noqa: SLF001
|
|
# Redefine Array to include both JAX arrays and PyTorch tensors
|
|
Array = jax.Array | torch.Tensor
|
|
|
|
|
|
-def _check_dataclass_annotations(self, typechecker):
|
|
- if not any(
|
|
- frame.frame.f_globals.get("__name__") in {"jax._src.tree_util", "flax.nnx.transforms.compilation"}
|
|
- for frame in inspect.stack()
|
|
- ):
|
|
- return _original_check_dataclass_annotations(self, typechecker)
|
|
- return None
|
|
+if _original_check_dataclass_annotations is not None:
|
|
|
|
+ def _check_dataclass_annotations(self, typechecker):
|
|
+ if not any(
|
|
+ frame.frame.f_globals.get("__name__") in {"jax._src.tree_util", "flax.nnx.transforms.compilation"}
|
|
+ for frame in inspect.stack()
|
|
+ ):
|
|
+ return _original_check_dataclass_annotations(self, typechecker)
|
|
+ return None
|
|
|
|
-jaxtyping._decorator._check_dataclass_annotations = _check_dataclass_annotations # noqa: SLF001
|
|
+ jaxtyping._decorator._check_dataclass_annotations = _check_dataclass_annotations # noqa: SLF001
|
|
|
|
KeyArrayLike: TypeAlias = jax.typing.ArrayLike
|
|
Params: TypeAlias = PyTree[Float[ArrayLike, "..."]]
|
|
diff --git a/src/openpi/shared/image_tools.py b/src/openpi/shared/image_tools.py
|
|
index 8cde353..6c5c034 100644
|
|
--- a/src/openpi/shared/image_tools.py
|
|
+++ b/src/openpi/shared/image_tools.py
|
|
@@ -90,8 +90,16 @@ def resize_with_pad_torch(
|
|
resized_width = int(cur_width / ratio)
|
|
|
|
# Resize
|
|
+ # 来源: openpi仓库 src/openpi/shared/image_tools.py @ 15a9616
|
|
+ # 改动点: 1) bilinear 时开 antialias=True, 对齐 golden (PIL BILINEAR 缩小时为抗锯齿
|
|
+ # scale-aware 三角核; jax.image.resize 默认 antialias=True 天然一致,
|
|
+ # F.interpolate 默认 False 导致缩小场景与 golden 大范围偏差)
|
|
resized_images = F.interpolate(
|
|
- images, size=(resized_height, resized_width), mode=mode, align_corners=False if mode == "bilinear" else None
|
|
+ images,
|
|
+ size=(resized_height, resized_width),
|
|
+ mode=mode,
|
|
+ align_corners=False if mode == "bilinear" else None,
|
|
+ antialias=True if mode == "bilinear" else False,
|
|
)
|
|
|
|
# Handle dtype-specific clipping
|
|
diff --git a/src/openpi/shared/nnx_utils.py b/src/openpi/shared/nnx_utils.py
|
|
index 29df222..5ea9bf9 100644
|
|
--- a/src/openpi/shared/nnx_utils.py
|
|
+++ b/src/openpi/shared/nnx_utils.py
|
|
@@ -65,5 +65,9 @@ class PathRegex:
|
|
|
|
def state_map(state: nnx.State, filter: nnx.filterlib.Filter, fn: Callable[[Any], Any]) -> nnx.State:
|
|
"""Apply a function to the leaves of the state that match the filter."""
|
|
- filtered_keys = set(state.filter(filter).flat_state())
|
|
- return state.map(lambda k, v: fn(v) if k in filtered_keys else v)
|
|
+ # 来源: openpi仓库 src/openpi/shared/nnx_utils.py @ 15a9616
|
|
+ # 改动点: 1) flax>=0.12 的 State.flat_state() 返回 FlatState 序列 (旧版 dict),
|
|
+ # set(...flat_state()) 对 Param 不可 hash 报错; 改为 to_predicate 逐叶判断,
|
|
+ # 两版语义一致
|
|
+ predicate = nnx.filterlib.to_predicate(filter)
|
|
+ return state.map(lambda k, v: fn(v) if predicate(tuple(k), v) else v)
|
|
diff --git a/src/openpi/shared/normalize.py b/src/openpi/shared/normalize.py
|
|
index 5a75fae..f56daea 100644
|
|
--- a/src/openpi/shared/normalize.py
|
|
+++ b/src/openpi/shared/normalize.py
|
|
@@ -12,6 +12,10 @@ class NormStats:
|
|
std: numpydantic.NDArray
|
|
q01: numpydantic.NDArray | None = None # 1st quantile
|
|
q99: numpydantic.NDArray | None = None # 99th quantile
|
|
+ # DataPipeline 改动 (design_v2 D4): 真 min/max 统计, 供 min_max 归一化模式。
|
|
+ # 旧 norm_stats.json 缺字段时为 None, 消费方回退直方图分位数并 INFO。
|
|
+ min: numpydantic.NDArray | None = None
|
|
+ max: numpydantic.NDArray | None = None
|
|
|
|
|
|
class RunningStats:
|
|
@@ -83,7 +87,8 @@ class RunningStats:
|
|
variance = self._mean_of_squares - self._mean**2
|
|
stddev = np.sqrt(np.maximum(0, variance))
|
|
q01, q99 = self._compute_quantiles([0.01, 0.99])
|
|
- return NormStats(mean=self._mean, std=stddev, q01=q01, q99=q99)
|
|
+ # DataPipeline 改动 (D4): 暴露真 min/max (RunningStats 内部本就维护)。
|
|
+ return NormStats(mean=self._mean, std=stddev, q01=q01, q99=q99, min=self._min, max=self._max)
|
|
|
|
def _adjust_histograms(self):
|
|
"""Adjust histograms when min or max changes."""
|
|
diff --git a/src/openpi/training/checkpoints.py b/src/openpi/training/checkpoints.py
|
|
index f32a831..fbdd572 100644
|
|
--- a/src/openpi/training/checkpoints.py
|
|
+++ b/src/openpi/training/checkpoints.py
|
|
@@ -41,8 +41,12 @@ def initialize_checkpoint_dir(
|
|
checkpoint_dir,
|
|
item_handlers={
|
|
"assets": CallbackHandler(),
|
|
- "train_state": ocp.PyTreeCheckpointHandler(),
|
|
- "params": ocp.PyTreeCheckpointHandler(),
|
|
+ # use_ocdbt=False: OCDBT 的 manifest 原子写语义在 NFS 类网络盘
|
|
+ # (平台 workspace PVC) 上会产生读回不一致 (DATA_LOSS: Length in
|
|
+ # header does not match); 关闭后走树状文件布局, 网络盘安全。
|
|
+ # 读取侧 orbax 自动识别格式, 官方 OCDBT 权重恢复不受影响。
|
|
+ "train_state": ocp.PyTreeCheckpointHandler(use_ocdbt=False),
|
|
+ "params": ocp.PyTreeCheckpointHandler(use_ocdbt=False),
|
|
},
|
|
options=ocp.CheckpointManagerOptions(
|
|
max_to_keep=1,
|
|
diff --git a/src/openpi/training/config.py b/src/openpi/training/config.py
|
|
index 4ca47e1..6b3e2d8 100644
|
|
--- a/src/openpi/training/config.py
|
|
+++ b/src/openpi/training/config.py
|
|
@@ -81,6 +81,14 @@ class DataConfig:
|
|
model_transforms: _transforms.Group = dataclasses.field(default_factory=_transforms.Group)
|
|
# If true, will use quantile normalization. Otherwise, normal z-score normalization will be used.
|
|
use_quantile_norm: bool = False
|
|
+ # DataPipeline 改动 (design_v2 D4): 契约归一化模式显式注入 (None -> 官方
|
|
+ # use_quantile_norm 行为): "min_max" / "mean_std" / "quantile"。
|
|
+ norm_mode: str | None = None
|
|
+
|
|
+ # DataPipeline 改动 (design_v2 多数据集聚合): 主 repo_id 之外的附加数据集
|
|
+ # repo_id 列表 (共父目录, 与主数据集共用同一套 transforms / norm_stats)。
|
|
+ # create_torch_dataset 会对主 + 附加逐一建库并 ConcatDataset 拼接。
|
|
+ extra_repo_ids: tuple[str, ...] = ()
|
|
|
|
# Names of keys that will be used by the data loader to generate the action sequence. The length of the
|
|
# sequence is defined by the `action_horizon` field in the model config. This should be adjusted if your
|
|
diff --git a/src/openpi/training/data_loader.py b/src/openpi/training/data_loader.py
|
|
index e2ee7dd..9f11269 100644
|
|
--- a/src/openpi/training/data_loader.py
|
|
+++ b/src/openpi/training/data_loader.py
|
|
@@ -1,4 +1,5 @@
|
|
from collections.abc import Iterator, Sequence
|
|
+import dataclasses
|
|
import logging
|
|
import multiprocessing
|
|
import os
|
|
@@ -7,7 +8,12 @@ from typing import Literal, Protocol, SupportsIndex, TypeVar
|
|
|
|
import jax
|
|
import jax.numpy as jnp
|
|
-import lerobot.common.datasets.lerobot_dataset as lerobot_dataset
|
|
+# DataPipeline 改动 (design_v2 D1): LeRobot v3 读取 (lerobot.datasets.* 命名空间,
|
|
+# 与 data2lerobot 转换器同大版本线 >=0.4.0); v2 安装时回退旧命名空间。
|
|
+try:
|
|
+ import lerobot.datasets.lerobot_dataset as lerobot_dataset
|
|
+except ImportError: # pragma: no cover - v2 fallback
|
|
+ import lerobot.common.datasets.lerobot_dataset as lerobot_dataset
|
|
import numpy as np
|
|
import torch
|
|
|
|
@@ -127,6 +133,28 @@ class FakeDataset(Dataset):
|
|
return self._num_samples
|
|
|
|
|
|
+def _create_single_lerobot_dataset(repo_id: str, data_config: _config.DataConfig, action_horizon: int):
|
|
+ """DataPipeline 改动 (多数据集聚合): 单 repo 的 LeRobot 数据集构建, 供主/附加
|
|
+ 数据集复用; delta_timestamps 的秒换算按各数据集自身 fps (fps 可不同)。"""
|
|
+ dataset_meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
|
|
+ dataset = lerobot_dataset.LeRobotDataset(
|
|
+ repo_id,
|
|
+ delta_timestamps={
|
|
+ key: [t / dataset_meta.fps for t in range(action_horizon)] for key in data_config.action_sequence_keys
|
|
+ },
|
|
+ )
|
|
+
|
|
+ if data_config.prompt_from_task:
|
|
+ # 改动点: lerobot >=0.4 的 meta.tasks 为 DataFrame(index=任务文本, 列含 task_index),
|
|
+ # PromptFromLeRobotTask 期望 dict[int, str] (0.1.0 语义), 此处归一化。
|
|
+ tasks = dataset_meta.tasks
|
|
+ if hasattr(tasks, "loc"): # pandas DataFrame
|
|
+ tasks = {int(row.task_index): idx for idx, row in tasks.iterrows()}
|
|
+ dataset = TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(tasks)])
|
|
+
|
|
+ return dataset
|
|
+
|
|
+
|
|
def create_torch_dataset(
|
|
data_config: _config.DataConfig, action_horizon: int, model_config: _model.BaseModelConfig
|
|
) -> Dataset:
|
|
@@ -137,16 +165,19 @@ def create_torch_dataset(
|
|
if repo_id == "fake":
|
|
return FakeDataset(model_config, num_samples=1024)
|
|
|
|
- dataset_meta = lerobot_dataset.LeRobotDatasetMetadata(repo_id)
|
|
- dataset = lerobot_dataset.LeRobotDataset(
|
|
- data_config.repo_id,
|
|
- delta_timestamps={
|
|
- key: [t / dataset_meta.fps for t in range(action_horizon)] for key in data_config.action_sequence_keys
|
|
- },
|
|
- )
|
|
+ dataset = _create_single_lerobot_dataset(repo_id, data_config, action_horizon)
|
|
|
|
- if data_config.prompt_from_task:
|
|
- dataset = TransformedDataset(dataset, [_transforms.PromptFromLeRobotTask(dataset_meta.tasks)])
|
|
+ # DataPipeline 改动 (多数据集聚合): 附加数据集与主数据集共用同一 DataConfig 的
|
|
+ # transforms / norm_stats (repo_id 逐个替换), ConcatDataset 顺序拼接。
|
|
+ extra_repo_ids = list(getattr(data_config, "extra_repo_ids", ()) or ())
|
|
+ if extra_repo_ids:
|
|
+ parts = [dataset]
|
|
+ for extra_id in extra_repo_ids:
|
|
+ sub_config = dataclasses.replace(data_config, repo_id=extra_id, extra_repo_ids=())
|
|
+ parts.append(_create_single_lerobot_dataset(extra_id, sub_config, action_horizon))
|
|
+ dataset = torch.utils.data.ConcatDataset(parts)
|
|
+ logging.info(f"multi-dataset concat: {len(parts)} datasets "
|
|
+ f"({', '.join([repo_id, *extra_repo_ids])}), total {len(dataset)} frames")
|
|
|
|
return dataset
|
|
|
|
@@ -180,12 +211,18 @@ def transform_dataset(dataset: Dataset, data_config: _config.DataConfig, *, skip
|
|
)
|
|
norm_stats = data_config.norm_stats
|
|
|
|
+ # DataPipeline 改动 (D4): norm_mode 显式注入 (None -> 官方 use_quantile_norm 行为)。
|
|
return TransformedDataset(
|
|
dataset,
|
|
[
|
|
*data_config.repack_transforms.inputs,
|
|
*data_config.data_transforms.inputs,
|
|
- _transforms.Normalize(norm_stats, use_quantiles=data_config.use_quantile_norm),
|
|
+ _transforms.Normalize(
|
|
+ norm_stats,
|
|
+ use_quantiles=data_config.use_quantile_norm,
|
|
+ mode=data_config.norm_mode,
|
|
+ clip_to_quantiles=data_config.norm_mode == "mean_std",
|
|
+ ),
|
|
*data_config.model_transforms.inputs,
|
|
],
|
|
)
|
|
diff --git a/src/openpi/training/sharding.py b/src/openpi/training/sharding.py
|
|
index 344c550..ec4f382 100644
|
|
--- a/src/openpi/training/sharding.py
|
|
+++ b/src/openpi/training/sharding.py
|
|
@@ -7,13 +7,37 @@ import numpy as np
|
|
BATCH_AXIS = "batch"
|
|
FSDP_AXIS = "fsdp"
|
|
# In FSDP, we shard the data across both the batch and FSDP axes.
|
|
+# DataPipeline 注记: jax>=0.11 下当参数维恰可被 fsdp 整除时 (如 6 卡的 1152),
|
|
+# 数据复合轴与参数 fsdp 切分混合的 contraction 会报非法 sharding (旧版 jax 隐式
|
|
+# all-gather 不再自动插入); LoRA/低显存路径用 FSDP_DEVICES=1 (数据并行, 参数
|
|
+# replicate) 回避, 全量 FSDP 路径需显式 reshard 改造 (待 80GB 卡场景处理)
|
|
DATA_AXIS = (BATCH_AXIS, FSDP_AXIS)
|
|
|
|
|
|
+def data_axis_spec(mesh: jax.sharding.Mesh) -> tuple[str, ...]:
|
|
+ """DataPipeline 改动: 数据分片轴统一为纯 batch 轴。
|
|
+
|
|
+ - 单设备: 空 spec (replicated) — jax>=0.11 Explicit mesh 下任何轴名标记
|
|
+ (即使份额为 1) 都会触发词表 embedding gather 的输出分片歧义断言;
|
|
+ - 多设备: 仅 'batch' 轴 (数据并行度 = device_count / fsdp_devices)。
|
|
+ 官方复合轴 (batch, fsdp) 在 jax>=0.11 下与参数 fsdp 切分混合的
|
|
+ contraction 不再自动插入 all-gather (DenseGeneral 输出 fsdp 双重
|
|
+ 占用 → 非法 sharding), 故数据不再沾 fsdp 轴; fsdp 轴仅用于参数。
|
|
+ """
|
|
+ if mesh.devices.size == 1:
|
|
+ return ()
|
|
+ return (BATCH_AXIS,)
|
|
+
|
|
+
|
|
class _MeshState:
|
|
active_mesh: jax.sharding.Mesh | None = None
|
|
|
|
|
|
+def active_mesh() -> jax.sharding.Mesh | None:
|
|
+ """DataPipeline 改动: 模型层 (gemma 词表 gather 的显式 out_sharding) 需要只读访问当前 mesh。"""
|
|
+ return _MeshState.active_mesh
|
|
+
|
|
+
|
|
def make_mesh(num_fsdp_devices: int) -> jax.sharding.Mesh:
|
|
if jax.device_count() % num_fsdp_devices != 0:
|
|
raise ValueError(
|
|
@@ -38,11 +62,26 @@ def set_mesh(mesh: jax.sharding.Mesh):
|
|
|
|
|
|
def activation_sharding_constraint(pytree):
|
|
- if _MeshState.active_mesh is None:
|
|
+ # DataPipeline 改动: jax>=0.11 Explicit mesh 下 with_sharding_constraint 是
|
|
+ # 断言 (静态 spec 无法匹配 contraction 的自然分片); 改用 jax.sharding.reshard
|
|
+ # 强制层边界激活布局 = 维度 0 对齐 batch 轴、特征维 replicated (层间 all-gather,
|
|
+ # 即 FSDP 的标准层边界语义; MLP 中间大激活保持 fsdp 切分省显存)。
|
|
+ # 这同时满足 scan 对 carry 进出同型的要求。
|
|
+ if _MeshState.active_mesh is None or _MeshState.active_mesh.shape[FSDP_AXIS] == 1:
|
|
return pytree
|
|
- return jax.lax.with_sharding_constraint(
|
|
- pytree, jax.sharding.NamedSharding(_MeshState.active_mesh, jax.sharding.PartitionSpec(DATA_AXIS))
|
|
- )
|
|
+ mesh = _MeshState.active_mesh
|
|
+
|
|
+ def _reshard(a):
|
|
+ if not hasattr(a, "shape") or not hasattr(a, "dtype"):
|
|
+ return a
|
|
+ # 1 维数组 (如 per-token 的 ar_mask [N]) 无 batch 维, 标记 dim0 为 batch
|
|
+ # 会在 broadcast 后污染目标维并阻塞 reduce; 只处理 dim0 为 batch 的数组
|
|
+ if len(a.shape) < 2:
|
|
+ return a
|
|
+ spec = jax.sharding.PartitionSpec(*((BATCH_AXIS,) + (None,) * (len(a.shape) - 1)))
|
|
+ return jax.sharding.reshard(a, jax.sharding.NamedSharding(mesh, spec))
|
|
+
|
|
+ return jax.tree_util.tree_map(_reshard, pytree)
|
|
|
|
|
|
def fsdp_sharding(
|
|
@@ -80,15 +119,35 @@ def fsdp_sharding(
|
|
if (arr_size := np.prod(array.shape) * np.dtype(array.dtype).itemsize) < min_size_bytes:
|
|
return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
|
|
|
|
- # shard matrices and larger tensors along the largest axis that is divisible by the fsdp dimension
|
|
- axes = np.argsort(array.shape)[::-1]
|
|
- spec = [None] * len(axes)
|
|
- for i in axes:
|
|
+ # DataPipeline 改动: attention 类参数一律 replicate。scan 将 27/18 层堆叠
|
|
+ # 为单个大权重, 阈值无法按层粒度放行; 而 attention 权重任何维度的切分
|
|
+ # 都会进入收缩维 (qkv 的 head_dim 在 MHA einsum 收缩; out proj 的输入
|
|
+ # 维在 attn_vec_einsum 收缩), jax>=0.11 下双边 fsdp 收缩必须显式
|
|
+ # out_sharding (flax MHA 内部不可注入), 故整体 replicate (约 1GB,
|
|
+ # 相对总参数是小头)。
|
|
+ _kp = jax.tree_util.keystr(kp)
|
|
+ if (
|
|
+ "MultiHeadDotProductAttention" in _kp
|
|
+ or "['attn']" in _kp
|
|
+ or "attn_vec_einsum" in _kp
|
|
+ or "q_einsum" in _kp
|
|
+ or "kv_einsum" in _kp
|
|
+ ):
|
|
+ return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
|
|
+
|
|
+ # DataPipeline 改动: 轴选择从"最大可整除轴优先"改为"最后维优先"。
|
|
+ # flax Dense kernel 为 [in, out], 收缩维是 dim0; 切最后维 (out 维) 时
|
|
+ # contraction 输出可自然分片, 不产生双边 fsdp 收缩歧义 (jax>=0.11 不再
|
|
+ # 自动插入 all-gather)。词表 embedding [vocab, d] 切 d 维后, gemma.encode
|
|
+ # 的 gather 以显式 out_sharding (d 维 replicated) 收敛, 即 FSDP 的
|
|
+ # 用时 all-gather 语义。
|
|
+ for i in reversed(range(len(array.shape))):
|
|
if array.shape[i] % mesh.shape[FSDP_AXIS] == 0:
|
|
if log:
|
|
logging.info(
|
|
f"Sharding {jax.tree_util.keystr(kp)} of shape {array.shape} ({arr_size / 2**20:.2f} MiB) along axis {i}"
|
|
)
|
|
+ spec = [None] * len(array.shape)
|
|
spec[i] = FSDP_AXIS
|
|
return jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(*spec))
|
|
|
|
diff --git a/src/openpi/training/utils.py b/src/openpi/training/utils.py
|
|
index fe7f94d..bc7d155 100644
|
|
--- a/src/openpi/training/utils.py
|
|
+++ b/src/openpi/training/utils.py
|
|
@@ -13,6 +13,12 @@ from openpi.shared import array_typing as at
|
|
@at.typecheck
|
|
@struct.dataclass
|
|
class TrainState:
|
|
+ # 来源: openpi仓库 src/openpi/training/utils.py @ 15a9616
|
|
+ # 改动点: 1) 设 __no_type_check__ 跳过 jaxtyped 对 dataclass __init__ 的运行时检查 —
|
|
+ # jaxtyping>=0.3 移除了 _check_dataclass_annotations 豁免挂点 (issue #277 上游重构),
|
|
+ # eval_shape/tracing 下 step 会被以 ShapeDtypeStruct / NamedSharding 等抽象占位
|
|
+ # 初始化, beartype 误拒; __no_type_check__ 是 jaxtyped 对 dataclass 的官方跳过开关
|
|
+ __no_type_check__ = True
|
|
step: at.Int[at.ArrayLike, ""]
|
|
params: nnx.State
|
|
model_def: nnx.GraphDef[_model.BaseModel]
|
|
diff --git a/src/openpi/transforms.py b/src/openpi/transforms.py
|
|
index 272375e..8764739 100644
|
|
--- a/src/openpi/transforms.py
|
|
+++ b/src/openpi/transforms.py
|
|
@@ -118,25 +118,43 @@ class Normalize(DataTransformFn):
|
|
use_quantiles: bool = False
|
|
# If true, will raise an error if any of the keys in the norm stats are not present in the data.
|
|
strict: bool = False
|
|
+ # DataPipeline 改动 (design_v2 D4/D19): 归一化模式显式化。
|
|
+ # None -> 由 use_quantiles 推断 (官方行为: quantile / mean_std);
|
|
+ # "min_max": 有界 -> [-1,1] (min_max 统计, 兼容 discrete_state_input, 契约默认);
|
|
+ # "quantile" / "mean_std": 与官方两档一致。
|
|
+ mode: str | None = None
|
|
+ # mean_std + discrete_state 注入时 clip 到 q01/q99 的归一化像 (D19, 对齐作者
|
|
+ # quantile 有界化意图), 避免 256-bin 离散化截断。
|
|
+ clip_to_quantiles: bool = False
|
|
|
|
def __post_init__(self):
|
|
- if self.norm_stats is not None and self.use_quantiles:
|
|
+ resolved_mode = self.mode or ("quantile" if self.use_quantiles else "mean_std")
|
|
+ if resolved_mode not in ("mean_std", "quantile", "min_max"):
|
|
+ raise ValueError(f"unsupported normalize mode: {resolved_mode}")
|
|
+ if self.norm_stats is not None and resolved_mode == "quantile":
|
|
_assert_quantile_stats(self.norm_stats)
|
|
+ object.__setattr__(self, "mode", resolved_mode)
|
|
|
|
def __call__(self, data: DataDict) -> DataDict:
|
|
if self.norm_stats is None:
|
|
return data
|
|
|
|
- return apply_tree(
|
|
- data,
|
|
- self.norm_stats,
|
|
- self._normalize_quantile if self.use_quantiles else self._normalize,
|
|
- strict=self.strict,
|
|
- )
|
|
+ fn = {
|
|
+ "mean_std": self._normalize,
|
|
+ "quantile": self._normalize_quantile,
|
|
+ "min_max": self._normalize_min_max,
|
|
+ }[self.mode]
|
|
+ return apply_tree(data, self.norm_stats, fn, strict=self.strict)
|
|
|
|
def _normalize(self, x, stats: NormStats):
|
|
mean, std = stats.mean[..., : x.shape[-1]], stats.std[..., : x.shape[-1]]
|
|
- return (x - mean) / (std + 1e-6)
|
|
+ out = (x - mean) / (std + 1e-6)
|
|
+ if self.clip_to_quantiles and stats.q01 is not None and stats.q99 is not None:
|
|
+ q01, q99 = stats.q01[..., : x.shape[-1]], stats.q99[..., : x.shape[-1]]
|
|
+ lo = (q01 - mean) / (std + 1e-6)
|
|
+ hi = (q99 - mean) / (std + 1e-6)
|
|
+ out = np.clip(out, lo, hi)
|
|
+ return out
|
|
|
|
def _normalize_quantile(self, x, stats: NormStats):
|
|
assert stats.q01 is not None
|
|
@@ -144,28 +162,44 @@ class Normalize(DataTransformFn):
|
|
q01, q99 = stats.q01[..., : x.shape[-1]], stats.q99[..., : x.shape[-1]]
|
|
return (x - q01) / (q99 - q01 + 1e-6) * 2.0 - 1.0
|
|
|
|
+ def _normalize_min_max(self, x, stats: NormStats):
|
|
+ lo = stats.min if stats.min is not None else stats.q01
|
|
+ hi = stats.max if stats.max is not None else stats.q99
|
|
+ assert lo is not None and hi is not None, (
|
|
+ "min_max mode requires min/max stats (recompute norm_stats); "
|
|
+ "legacy stats fallback to q01/q99"
|
|
+ )
|
|
+ lo, hi = lo[..., : x.shape[-1]], hi[..., : x.shape[-1]]
|
|
+ return (x - lo) / (hi - lo + 1e-6) * 2.0 - 1.0
|
|
+
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Unnormalize(DataTransformFn):
|
|
norm_stats: at.PyTree[NormStats] | None
|
|
# If true, will use quantile normalization. Otherwise, normal z-score normalization will be used.
|
|
use_quantiles: bool = False
|
|
+ # DataPipeline 改动 (D4): 与 Normalize.mode 对称。
|
|
+ mode: str | None = None
|
|
|
|
def __post_init__(self):
|
|
- if self.norm_stats is not None and self.use_quantiles:
|
|
+ resolved_mode = self.mode or ("quantile" if self.use_quantiles else "mean_std")
|
|
+ if resolved_mode not in ("mean_std", "quantile", "min_max"):
|
|
+ raise ValueError(f"unsupported unnormalize mode: {resolved_mode}")
|
|
+ if self.norm_stats is not None and resolved_mode == "quantile":
|
|
_assert_quantile_stats(self.norm_stats)
|
|
+ object.__setattr__(self, "mode", resolved_mode)
|
|
|
|
def __call__(self, data: DataDict) -> DataDict:
|
|
if self.norm_stats is None:
|
|
return data
|
|
|
|
+ fn = {
|
|
+ "mean_std": self._unnormalize,
|
|
+ "quantile": self._unnormalize_quantile,
|
|
+ "min_max": self._unnormalize_min_max,
|
|
+ }[self.mode]
|
|
# Make sure that all the keys in the norm stats are present in the data.
|
|
- return apply_tree(
|
|
- data,
|
|
- self.norm_stats,
|
|
- self._unnormalize_quantile if self.use_quantiles else self._unnormalize,
|
|
- strict=True,
|
|
- )
|
|
+ return apply_tree(data, self.norm_stats, fn, strict=True)
|
|
|
|
def _unnormalize(self, x, stats: NormStats):
|
|
mean = pad_to_dim(stats.mean, x.shape[-1], axis=-1, value=0.0)
|
|
@@ -180,6 +214,16 @@ class Unnormalize(DataTransformFn):
|
|
return np.concatenate([(x[..., :dim] + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01, x[..., dim:]], axis=-1)
|
|
return (x + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01
|
|
|
|
+ def _unnormalize_min_max(self, x, stats: NormStats):
|
|
+ lo = stats.min if stats.min is not None else stats.q01
|
|
+ hi = stats.max if stats.max is not None else stats.q99
|
|
+ assert lo is not None and hi is not None
|
|
+ if (dim := lo.shape[-1]) < x.shape[-1]:
|
|
+ return np.concatenate(
|
|
+ [(x[..., :dim] + 1.0) / 2.0 * (hi - lo + 1e-6) + lo, x[..., dim:]], axis=-1
|
|
+ )
|
|
+ return (x + 1.0) / 2.0 * (hi - lo + 1e-6) + lo
|
|
+
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class ResizeImages(DataTransformFn):
|
|
@@ -324,6 +368,44 @@ class PromptFromLeRobotTask(DataTransformFn):
|
|
return {**data, "prompt": prompt}
|
|
|
|
|
|
+@dataclasses.dataclass(frozen=True)
|
|
+class ImagesToHWC(DataTransformFn):
|
|
+ """DataPipeline 改动 (design_v2 D1): 对齐官方 libero_policy._parse_image 语义 —
|
|
+ lerobot >=0.4 视频帧为 float32 [0,1] CHW, 官方数据管线为 uint8 HWC
|
|
+ (ResizeImages 走 PIL uint8 golden; Observation.from_dict 再转 [-1,1] float,
|
|
+ torch 分支并 permute CHW)。float 输入 clip [0,1] 后 *255 取整, 对
|
|
+ uint8/HWC 输入幂等无操作。
|
|
+ """
|
|
+
|
|
+ def __call__(self, data: DataDict) -> DataDict:
|
|
+ images = data.get("image")
|
|
+ if not isinstance(images, dict):
|
|
+ return data
|
|
+
|
|
+ def _parse(img):
|
|
+ img = np.asarray(img)
|
|
+ if np.issubdtype(img.dtype, np.floating):
|
|
+ img = (255 * np.clip(img, 0.0, 1.0)).astype(np.uint8)
|
|
+ if img.ndim == 3 and img.shape[0] <= 4 < img.shape[-1]:
|
|
+ img = np.transpose(img, (1, 2, 0))
|
|
+ return img
|
|
+
|
|
+ return {**data, "image": {key: _parse(value) for key, value in images.items()}}
|
|
+
|
|
+
|
|
+@dataclasses.dataclass(frozen=True)
|
|
+class BuildImageMasks(DataTransformFn):
|
|
+ """DataPipeline 改动 (design_v2 D15): 为契约 N 槽相机生成 image_mask (每槽 True)。
|
|
+
|
|
+ 槽位完备性由契约 key_map 校验保证 (每个声明相机恰好被映射一次), 无需官方
|
|
+ aloha_policy 式缺路填黑 + False 分支。缺此 transform 时 Observation.from_dict
|
|
+ 取 data["image_mask"] 抛 KeyError。
|
|
+ """
|
|
+
|
|
+ def __call__(self, data: DataDict) -> DataDict:
|
|
+ return {**data, "image_mask": {key: np.True_ for key in data["image"]}}
|
|
+
|
|
+
|
|
@dataclasses.dataclass(frozen=True)
|
|
class PadStatesAndActions(DataTransformFn):
|
|
"""Zero-pads states and actions to the model action dimension."""
|
|
--
|
|
2.34.1
|
|
|