614 lines
28 KiB
Python
614 lines
28 KiB
Python
# 来源: openpi 仓库 scripts/train.py @ 15a9616 (JAX 训练主循环拷贝改造)
|
|
# 改动点:
|
|
# 1) 入口从 tyro CLI 配置名改为超参环境变量 + CONTRACT_JSON 契约 (pi_pipeline.contract),
|
|
# TrainConfig 由 build_train_config() 程序化组装, 不注册进 _CONFIGS
|
|
# 2) wandb 强制关闭 (design_v2 §10.2: 离线镜像 wandb 禁用), 首批图像 wandb 日志移除
|
|
# 3) 训练循环按 log_interval 打 DROBOTICFLOW_PROGRESS (RFC3339 updatedAt)
|
|
# 4) norm_stats 缺失时自动补算 (幂等: 存在即跳过, D11); 优先级 上游 NORM_STATS_PATH 显式
|
|
# 指定 > 自动计算
|
|
# 5) checkpoint 收尾写 model_structure.json (契约快照, 量化/serve/convert 锚点, D13/R-9)
|
|
# 与 $OUTPUT_DIR/.artifacts.json
|
|
# 6) 训练入口超参: DATASET_PATH (与上游端口二选一上游优先) / CONTRACT_JSON / KEY_MAP /
|
|
# RESUME_CKPT / NORM_STATS_PATH / BATCH_SIZE / NUM_TRAIN_STEPS / LR_WARMUP_STEPS /
|
|
# LR_PEAK / LR_DECAY_STEPS / SAVE_INTERVAL / KEEP_PERIOD / SEED / NUM_WORKERS / RESUME /
|
|
# OVERWRITE / EXP_NAME
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import functools
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import etils.epath as epath # noqa: F401 (openpi 训练组件依赖此命名空间可用)
|
|
import flax.nnx as nnx
|
|
from flax.training import common_utils
|
|
import jax
|
|
import jax.numpy as jnp
|
|
import numpy as np
|
|
import optax
|
|
import tqdm_loggable.auto as tqdm
|
|
|
|
import openpi.models.model as _model
|
|
import openpi.shared.normalize as _normalize
|
|
import openpi.shared.nnx_utils as nnx_utils
|
|
import openpi.training.checkpoints as _checkpoints
|
|
import openpi.training.config as _config
|
|
import openpi.training.data_loader as _data_loader
|
|
import openpi.training.optimizer as _optimizer
|
|
import openpi.training.sharding as sharding
|
|
import openpi.training.utils as training_utils
|
|
import openpi.training.weight_loaders as _weight_loaders
|
|
import openpi.transforms as _transforms
|
|
from openpi.models.pi0_config import Pi0Config
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from pi_pipeline import contract as _contract # noqa: E402
|
|
from ws_common import input_locate, report_utils # noqa: E402
|
|
|
|
# 镜像内置官方权重 (构建期下载, D8: 只打 JAX pi05_base 一份)
|
|
OFFICIAL_PI05_BASE = "/app/src/openpi_weights/pi05_base/params"
|
|
|
|
logger = logging.getLogger("openpi")
|
|
|
|
|
|
def _env_int(key: str, default: int) -> int:
|
|
raw = os.environ.get(key, "").strip()
|
|
if not raw:
|
|
return default
|
|
try:
|
|
return int(raw)
|
|
except ValueError:
|
|
sys.exit(f"[hyperparam] {key}: invalid int value {raw!r}")
|
|
|
|
|
|
def _env_float(key: str, default: float) -> float:
|
|
raw = os.environ.get(key, "").strip()
|
|
if not raw:
|
|
return default
|
|
try:
|
|
return float(raw)
|
|
except ValueError:
|
|
sys.exit(f"[hyperparam] {key}: invalid float value {raw!r}")
|
|
|
|
|
|
# ============================================================
|
|
# 契约驱动的 DataConfigFactory
|
|
# ============================================================
|
|
|
|
|
|
class ContractDataConfigFactory(_config.DataConfigFactory):
|
|
"""把契约 key_map 编译成 repack transforms + 归一化模式的 DataConfig。
|
|
|
|
多数据集聚合: dataset_paths 为 1..N 个共父目录的数据集根; 所有数据集共用
|
|
同一套契约 transforms 与同一份 norm_stats (统计在全量混合上计算), 主
|
|
repo_id = 第一个数据集目录名, 其余进 extra_repo_ids (data_loader 侧
|
|
ConcatDataset 拼接)。asset_id 单数据集保持目录名 (向后兼容旧统计产物),
|
|
多数据集用确定性的混合 id。"""
|
|
|
|
def __init__(self, dataset_paths: list[Path], contract: _contract.Contract, assets_dir: Path):
|
|
if not dataset_paths:
|
|
raise ValueError("dataset_paths must be non-empty")
|
|
super().__init__(repo_id=dataset_paths[0].name)
|
|
self._dataset_paths = dataset_paths
|
|
self._contract = contract
|
|
self._assets_dir = assets_dir
|
|
|
|
@property
|
|
def asset_id(self) -> str:
|
|
names = [p.name for p in self._dataset_paths]
|
|
if len(names) == 1:
|
|
return names[0]
|
|
digest = __import__("hashlib").md5("|".join(names).encode()).hexdigest()[:8]
|
|
return f"mix{len(names)}-{digest}"
|
|
|
|
def create(self, assets_dirs, model_config: _model.BaseModelConfig) -> _config.DataConfig:
|
|
del assets_dirs # 数据集与统计均锚定显式路径, 不走全局 assets 目录
|
|
c = self._contract
|
|
|
|
# repack structure: 旧键扁平路径 -> 新键 (image/<slot>, state, actions);
|
|
# prompt 键透传 (PromptFromLeRobotTask 在 dataset 层已产出 "prompt",
|
|
# repack 重建 dict 时需显式保留, 否则 TokenizePrompt 报 Prompt is required)
|
|
image_struct = {slot: src for src, slots in c.vision_map.items() for slot in slots}
|
|
state_src = next(k for k in ("state/concat", "state", "observation.state") if k in c.key_map)
|
|
action_src = next(k for k in ("action/concat", "action", "actions") if k in c.key_map)
|
|
structure = {
|
|
"image": image_struct,
|
|
"state": state_src,
|
|
"actions": action_src,
|
|
"prompt": "prompt",
|
|
}
|
|
|
|
use_quantile = c.use_quantile_norm
|
|
norm_mode = None if c.norm_mode == "quantile" else c.norm_mode
|
|
|
|
return dataclasses.replace(
|
|
self.base_config or _config.DataConfig(),
|
|
repo_id=self._dataset_paths[0].name,
|
|
extra_repo_ids=tuple(p.name for p in self._dataset_paths[1:]),
|
|
asset_id=self.asset_id,
|
|
norm_stats=self._load_norm_stats(),
|
|
repack_transforms=_transforms.Group(inputs=[_transforms.RepackTransform(structure)]),
|
|
data_transforms=_transforms.Group(inputs=[_transforms.ImagesToHWC(), _transforms.BuildImageMasks()]),
|
|
# model_transforms: TokenizePrompt / ResizeImages / PadStatesAndActions
|
|
# (缺失时 jax 静默丢语言分支, torch 在 lang_embed 报 None)
|
|
model_transforms=_config.ModelTransformFactory()(model_config),
|
|
use_quantile_norm=use_quantile,
|
|
norm_mode=norm_mode,
|
|
action_sequence_keys=(action_src,),
|
|
prompt_from_task=True,
|
|
)
|
|
|
|
def dataset_configs(self, model_config: _model.BaseModelConfig) -> list[_config.DataConfig]:
|
|
"""每数据集一份 DataConfig (repo_id 各自, extra 为空, 其余共享)。
|
|
|
|
消费方: norm_stats 统计 (在全量混合上计算共享统计)。"""
|
|
base = self.create(None, model_config)
|
|
configs = [dataclasses.replace(base, repo_id=p.name, extra_repo_ids=()) for p in self._dataset_paths]
|
|
return configs
|
|
|
|
def _load_norm_stats(self):
|
|
stats_dir = self._assets_dir / self.asset_id
|
|
try:
|
|
stats = _normalize.load(stats_dir)
|
|
logging.info(f"[norm_stats] loaded from {stats_dir}")
|
|
return stats
|
|
except FileNotFoundError:
|
|
logging.info(f"[norm_stats] not found in {stats_dir} (will be computed)")
|
|
return None
|
|
|
|
|
|
# ============================================================
|
|
# TrainConfig 组装
|
|
# ============================================================
|
|
|
|
|
|
def resolve_dataset_paths() -> list[Path]:
|
|
"""输入根 (上游端口 portId=dataset 优先, DATASET_PATH 兜底) -> 数据集根列表。
|
|
|
|
根可为单数据集根或多数据集父目录 (expand_dataset_roots 展开); 多数据集
|
|
必须共父目录 (HF_LEROBOT_HOME 唯一)。每个数据集根校验 meta/info.json。
|
|
平台 workspace (PVC) 输入先整体暂存到节点本地盘再展开 (stage_to_local)。"""
|
|
root = input_locate.locate_input_dir(port_id="dataset", env_key="DATASET_PATH")
|
|
root = input_locate.stage_to_local(root)
|
|
paths = input_locate.expand_dataset_roots(root)
|
|
parents = {p.parent for p in paths}
|
|
if len(parents) != 1:
|
|
sys.exit(f"[input] datasets must share one parent dir for LeRobot local read, got {parents}")
|
|
for path in paths:
|
|
if not (path / "meta" / "info.json").is_file():
|
|
sys.exit(
|
|
f"[input] {path} is not a LeRobot v3 dataset root "
|
|
"(expected meta/info.json + data/*.parquet); got contents: "
|
|
f"{sorted(p.name for p in path.iterdir())[:10]}"
|
|
)
|
|
return paths
|
|
|
|
|
|
def build_train_config(contract: _contract.Contract, dataset_paths: list[Path], output_dir: Path) -> _config.TrainConfig:
|
|
resume_ckpt = os.environ.get("RESUME_CKPT", "").strip()
|
|
if resume_ckpt in ("", "official"):
|
|
weight_loader: _weight_loaders.WeightLoader
|
|
if os.path.isdir(OFFICIAL_PI05_BASE):
|
|
weight_loader = _weight_loaders.CheckpointWeightLoader(OFFICIAL_PI05_BASE)
|
|
else:
|
|
weight_loader = _weight_loaders.CheckpointWeightLoader("gs://openpi-assets/checkpoints/pi05_base/params")
|
|
else:
|
|
if not os.path.isdir(resume_ckpt):
|
|
sys.exit(f"[hyperparam] RESUME_CKPT directory not found: {resume_ckpt}")
|
|
weight_loader = _weight_loaders.CheckpointWeightLoader(resume_ckpt)
|
|
|
|
assets_dir = output_dir / "assets"
|
|
exp_name = os.environ.get("EXP_NAME", "").strip() or "pi05_contract"
|
|
|
|
lr_peak = _env_float("LR_PEAK", 2.5e-5)
|
|
# LORA=true: 官方 low_mem LoRA 微调惯例 (config.py pi0_libero_low_mem_finetune 同款):
|
|
# LoRA variant + freeze 非 LoRA 参数 (bf16) + 关 EMA。32GB 卡全量微调 init 峰值
|
|
# (params+EMA+AdamW 状态 replicated 中间量) 不可行时的冒烟/低显存路径。
|
|
use_lora = os.environ.get("LORA", "").strip().lower() in ("1", "true", "yes")
|
|
model_kwargs = dict(contract.pi0_config_kwargs())
|
|
if use_lora:
|
|
model_kwargs["paligemma_variant"] = "gemma_2b_lora"
|
|
model_kwargs["action_expert_variant"] = "gemma_300m_lora"
|
|
config = _config.TrainConfig(
|
|
name="pi05_contract",
|
|
project_name="datapipeline",
|
|
exp_name=exp_name,
|
|
model=Pi0Config(**model_kwargs),
|
|
weight_loader=weight_loader,
|
|
data=ContractDataConfigFactory(dataset_paths, contract, assets_dir / "pi05_contract"),
|
|
assets_base_dir=assets_dir,
|
|
# PI_CKPT_LOCAL=1: checkpoint 先写节点本地盘, 训练完成后再整体同步回
|
|
# OUTPUT_DIR/checkpoints。平台 workspace 为网络 PVC, orbax/tensorstore 的
|
|
# OCDBT manifest 原子写与 zarr .__lock 文件锁在其元数据一致性下会损坏/
|
|
# 失败; 本地直写 (本地盘部署) 保持原行为不受影响。
|
|
checkpoint_base_dir=(
|
|
Path(tempfile.gettempdir()) / "pi-ckpt" / "checkpoints"
|
|
if os.environ.get("PI_CKPT_LOCAL", "").strip().lower() in ("1", "true", "yes")
|
|
else output_dir / "checkpoints"
|
|
),
|
|
seed=_env_int("SEED", 0),
|
|
batch_size=_env_int("BATCH_SIZE", 8),
|
|
num_workers=_env_int("NUM_WORKERS", 4),
|
|
num_train_steps=_env_int("NUM_TRAIN_STEPS", 20000),
|
|
fsdp_devices=_env_int("FSDP_DEVICES", 1),
|
|
log_interval=_env_int("LOG_INTERVAL", 100),
|
|
save_interval=_env_int("SAVE_INTERVAL", 1000),
|
|
keep_period=_env_int("KEEP_PERIOD", 5000),
|
|
overwrite=os.environ.get("OVERWRITE", "").strip().lower() in ("1", "true", "yes"),
|
|
resume=os.environ.get("RESUME", "").strip().lower() in ("1", "true", "yes"),
|
|
wandb_enabled=False, # 改动点 2: 强制关闭
|
|
freeze_filter=Pi0Config(**model_kwargs).get_freeze_filter() if use_lora else nnx.Nothing(),
|
|
ema_decay=None if use_lora else _env_float("EMA_DECAY", 0.99),
|
|
lr_schedule=_optimizer.CosineDecaySchedule(
|
|
warmup_steps=_env_int("LR_WARMUP_STEPS", 1000),
|
|
peak_lr=lr_peak,
|
|
decay_steps=_env_int("LR_DECAY_STEPS", 30000),
|
|
),
|
|
optimizer=_optimizer.AdamW(),
|
|
)
|
|
logging.info(f"[contract] cameras={contract.cameras} "
|
|
f"resolution={contract.per_cam_resolution} prefix_tokens={contract.prefix_tokens} "
|
|
f"norm_mode={contract.norm_mode}")
|
|
return config
|
|
|
|
|
|
# ============================================================
|
|
# norm_stats 自动补算 (D11: 幂等, 存在即跳过)
|
|
# ============================================================
|
|
|
|
|
|
def compute_norm_stats_if_missing(config: _config.TrainConfig, contract: _contract.Contract) -> None:
|
|
explicit = os.environ.get("NORM_STATS_PATH", "").strip()
|
|
stats_dir = config.assets_dirs / config.data.asset_id
|
|
if explicit:
|
|
if not os.path.isfile(os.path.join(explicit, "norm_stats.json")):
|
|
sys.exit(f"[norm_stats] NORM_STATS_PATH has no norm_stats.json: {explicit}")
|
|
# 显式指定优先: 复制到 assets 目录供 DataConfig 加载
|
|
import shutil
|
|
|
|
shutil.copytree(explicit, stats_dir, dirs_exist_ok=True)
|
|
logging.info(f"[norm_stats] copied from explicit NORM_STATS_PATH {explicit} -> {stats_dir}")
|
|
return
|
|
|
|
if (stats_dir / "norm_stats.json").is_file():
|
|
logging.info(f"[norm_stats] exists, skip (idempotent): {stats_dir}")
|
|
return
|
|
|
|
# 多数据集: 统计在全量混合上计算 (共享一份 norm_stats, 全部数据集在
|
|
# repack+data_transform 空间逐帧累积)
|
|
logging.info(f"[norm_stats] computing in repack+data_transform space -> {stats_dir}")
|
|
stats = {"state": _normalize.RunningStats(), "actions": _normalize.RunningStats()}
|
|
total_frames = 0
|
|
for data_config in config.data.dataset_configs(config.model):
|
|
dataset = _data_loader.create_torch_dataset(data_config, config.model.action_horizon, config.model)
|
|
dataset = _data_loader.TransformedDataset(
|
|
dataset,
|
|
[
|
|
*data_config.repack_transforms.inputs,
|
|
*data_config.data_transforms.inputs,
|
|
_transforms.InjectDefaultPrompt(None),
|
|
],
|
|
)
|
|
for i in range(len(dataset)):
|
|
item = dataset[i]
|
|
for key in ("state", "actions"):
|
|
stats[key].update(np.asarray(item[key])[None, :] if key == "state" else np.asarray(item[key]))
|
|
if (total_frames + i + 1) % 500 == 0:
|
|
logging.info(f"[norm_stats] {data_config.repo_id}: {i + 1}/{len(dataset)} frames")
|
|
total_frames += len(dataset)
|
|
# 多数据集 asset 目录: norm_stats 落在共享 asset_id 下
|
|
stats_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
_normalize.save(stats_dir, {k: v.get_statistics() for k, v in stats.items()})
|
|
logging.info(f"[norm_stats] saved ({total_frames}+ frames): {stats_dir}")
|
|
|
|
|
|
# ============================================================
|
|
# 训练主循环 (来源: scripts/train.py main, 改动点见文件头)
|
|
# ============================================================
|
|
|
|
|
|
def _emit_progress(stage: str, **fields) -> None:
|
|
payload = {"stage": stage, **fields}
|
|
payload["updatedAt"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
print("DROBOTICFLOW_PROGRESS " + json.dumps(payload), flush=True)
|
|
|
|
|
|
def _adapt_action_dim(params: dict, action_dim: int) -> dict:
|
|
"""D5 / design_v2 §5.2: action_dim resume 适配 — action_in/out_proj kernel 循环 tile。
|
|
|
|
官方 pi05_base 的 action_dim=32; 契约维 ad != 32 时 (ad<32 截取, ad>32 循环重复)
|
|
适配到模型结构。bias/out_features 不受影响; state 走 tokenizer 无投影权重。
|
|
"""
|
|
for proj, axis in (("action_in_proj", 0), ("action_out_proj", 1)):
|
|
node = params.get(proj)
|
|
if not isinstance(node, dict) or "kernel" not in node:
|
|
continue
|
|
kernel = np.asarray(node["kernel"])
|
|
src_dim = kernel.shape[axis]
|
|
if src_dim == action_dim:
|
|
continue
|
|
reps = -(-action_dim // src_dim)
|
|
tiled = np.tile(kernel, (reps, 1) if axis == 0 else (1, reps))
|
|
tiled = tiled[:action_dim] if axis == 0 else tiled[:, :action_dim]
|
|
node["kernel"] = tiled
|
|
logging.info(f"[resume-adapt] {proj}.kernel {kernel.shape} -> {tiled.shape} (cyclic tile, D5)")
|
|
# out_proj 的 bias 维度 = action_dim (in_proj 的 bias 是 out_features 不受影响)
|
|
if "bias" in node and np.asarray(node["bias"]).shape[:1] == (src_dim,):
|
|
bias = np.asarray(node["bias"])
|
|
node["bias"] = np.tile(bias, reps)[:action_dim]
|
|
logging.info(f"[resume-adapt] {proj}.bias {bias.shape} -> {node['bias'].shape} (cyclic tile, D5)")
|
|
return params
|
|
|
|
|
|
def _load_weights_and_validate(loader, params_shape):
|
|
loaded_params = loader.load(params_shape)
|
|
import flax.traverse_util as traverse_util
|
|
|
|
unflattened = traverse_util.unflatten_dict(
|
|
{k: v for k, v in traverse_util.flatten_dict(loaded_params).items() if not isinstance(v, jax.ShapeDtypeStruct)}
|
|
)
|
|
return unflattened
|
|
|
|
|
|
def init_train_state(config, init_rng, mesh, *, resume):
|
|
tx = _optimizer.create_optimizer(config.optimizer, config.lr_schedule, weight_decay_mask=None)
|
|
|
|
def init(rng, partial_params=None):
|
|
rng, model_rng = jax.random.split(rng)
|
|
model = config.model.create(model_rng)
|
|
if partial_params is not None:
|
|
graphdef, state = nnx.split(model)
|
|
state.replace_by_pure_dict(partial_params)
|
|
model = nnx.merge(graphdef, state)
|
|
params = nnx.state(model)
|
|
params = nnx_utils.state_map(params, config.freeze_filter, lambda p: p.replace(p.value.astype(jnp.bfloat16)))
|
|
return training_utils.TrainState(
|
|
step=0,
|
|
params=params,
|
|
model_def=nnx.graphdef(model),
|
|
tx=tx,
|
|
opt_state=tx.init(params.filter(config.trainable_filter)),
|
|
ema_decay=config.ema_decay,
|
|
ema_params=None if config.ema_decay is None else params,
|
|
)
|
|
|
|
train_state_shape = jax.eval_shape(init, init_rng)
|
|
# DataPipeline 改动: 阈值 4->32 MiB — attention qkv/out proj 等中小权重在
|
|
# jax>=0.11 下切分 (即使 out 维) 仍会在 MHA einsum 的 head 维收缩处产生
|
|
# 双边 fsdp 歧义; replicate 中小权重, 只切 embedding / gemma expert MLP 等
|
|
# 大块 (显存大头), 兼顾收缩安全与省显存
|
|
state_sharding = sharding.fsdp_sharding(train_state_shape, mesh, min_size_mbytes=32, log=True)
|
|
|
|
if resume:
|
|
return train_state_shape, state_sharding
|
|
|
|
partial_params = _load_weights_and_validate(config.weight_loader, train_state_shape.params.to_pure_dict())
|
|
partial_params = _adapt_action_dim(partial_params, config.model.action_dim)
|
|
# LoRA (freeze_filter 生效): partial 全为冻结权重, init 的 state_map 本就转 bf16;
|
|
# 提前在 jit 外转可将 replicated 输入峰值 12GB(fp32) 降至 6GB, 结果 bit 级等效,
|
|
# 为 32GB 卡的 FSDP all-gather 留头寸 (D14 单机多卡约束下的显存适配)
|
|
if not isinstance(config.freeze_filter, nnx.Nothing):
|
|
partial_params = jax.tree.map(
|
|
lambda x: x.astype(np.float32).astype(jnp.bfloat16) if np.asarray(x).dtype == np.float32 else x, partial_params
|
|
)
|
|
replicated_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
|
|
|
|
train_state = jax.jit(
|
|
init,
|
|
donate_argnums=(1,),
|
|
in_shardings=replicated_sharding,
|
|
out_shardings=state_sharding,
|
|
)(init_rng, partial_params)
|
|
|
|
return train_state, state_sharding
|
|
|
|
|
|
def train_step(config, rng, state, batch):
|
|
model = nnx.merge(state.model_def, state.params)
|
|
model.train()
|
|
observation, actions = batch
|
|
|
|
def loss_fn(model, rng, observation, actions):
|
|
chunked_loss = model.compute_loss(rng, observation, actions, train=True)
|
|
return jnp.mean(chunked_loss)
|
|
|
|
train_rng = jax.random.fold_in(rng, state.step)
|
|
observation, actions = batch
|
|
|
|
diff_state = nnx.DiffState(0, config.trainable_filter)
|
|
loss, grads = nnx.value_and_grad(loss_fn, argnums=diff_state)(model, train_rng, observation, actions)
|
|
|
|
params = state.params.filter(config.trainable_filter)
|
|
updates, new_opt_state = state.tx.update(grads, state.opt_state, params)
|
|
new_params = optax.apply_updates(params, updates)
|
|
|
|
nnx.update(model, new_params)
|
|
new_params = nnx.state(model)
|
|
|
|
new_state = dataclasses.replace(state, step=state.step + 1, params=new_params, opt_state=new_opt_state)
|
|
if state.ema_decay is not None:
|
|
new_state = dataclasses.replace(
|
|
new_state,
|
|
ema_params=jax.tree.map(
|
|
lambda old, new: state.ema_decay * old + (1 - state.ema_decay) * new, state.ema_params, new_params
|
|
),
|
|
)
|
|
|
|
kernel_params = nnx.state(
|
|
model,
|
|
nnx.All(
|
|
nnx.Param,
|
|
nnx.Not(nnx_utils.PathRegex(".*/(bias|scale|pos_embedding|input_embedding)")),
|
|
lambda _, x: x.value.ndim > 1,
|
|
),
|
|
)
|
|
info = {
|
|
"loss": loss,
|
|
"grad_norm": optax.global_norm(grads),
|
|
"param_norm": optax.global_norm(kernel_params),
|
|
}
|
|
return new_state, info
|
|
|
|
|
|
def main(config: _config.TrainConfig, contract: _contract.Contract, output_dir: Path):
|
|
logging.info(f"Running on: {jax.devices()}")
|
|
|
|
# jax>=0.11: 数据只在 batch 轴切 (数据并行度 = device_count / fsdp_devices),
|
|
# 整除性按 batch 轴校验, 不再按总设备数 (官方复合轴语义的遗留校验)
|
|
data_parallelism = jax.device_count() // max(1, config.fsdp_devices)
|
|
if config.batch_size % data_parallelism != 0:
|
|
raise ValueError(
|
|
f"Batch size {config.batch_size} must be divisible by the data parallelism {data_parallelism} "
|
|
f"(devices {jax.device_count()} / fsdp {config.fsdp_devices})."
|
|
)
|
|
|
|
rng = jax.random.key(config.seed)
|
|
train_rng, init_rng = jax.random.split(rng)
|
|
|
|
mesh = sharding.make_mesh(config.fsdp_devices)
|
|
data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.data_axis_spec(mesh)))
|
|
replicated_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
|
|
|
|
checkpoint_manager, resuming = _checkpoints.initialize_checkpoint_dir(
|
|
config.checkpoint_dir,
|
|
keep_period=config.keep_period,
|
|
overwrite=config.overwrite,
|
|
resume=config.resume,
|
|
)
|
|
|
|
data_loader = _data_loader.create_data_loader(config, sharding=data_sharding, shuffle=True)
|
|
data_iter = iter(data_loader)
|
|
batch = next(data_iter)
|
|
logging.info(f"Initialized data loader:\n{training_utils.array_tree_to_info(batch)}")
|
|
|
|
train_state, train_state_sharding = init_train_state(config, init_rng, mesh, resume=resuming)
|
|
jax.block_until_ready(train_state)
|
|
logging.info(f"Initialized train state:\n{training_utils.array_tree_to_info(train_state.params)}")
|
|
|
|
if resuming:
|
|
train_state = _checkpoints.restore_state(checkpoint_manager, train_state, data_loader)
|
|
|
|
# jax>=0.11: jit in_shardings 必须与实际数组布局一致 (dataloader 已按
|
|
# data_sharding 布局产出); 词表 gather 的歧义已由 gemma.encode 的显式
|
|
# out_sharding 解决, 不再需要 prompt replicated 特例 (单卡时代遗留)
|
|
batch_in_shardings = data_sharding
|
|
ptrain_step = jax.jit(
|
|
functools.partial(train_step, config),
|
|
in_shardings=(replicated_sharding, train_state_sharding, batch_in_shardings),
|
|
out_shardings=(train_state_sharding, replicated_sharding),
|
|
donate_argnums=(1,),
|
|
)
|
|
|
|
start_step = int(train_state.step)
|
|
pbar = tqdm.tqdm(
|
|
range(start_step, config.num_train_steps),
|
|
initial=start_step,
|
|
total=config.num_train_steps,
|
|
dynamic_ncols=True,
|
|
)
|
|
|
|
infos = []
|
|
for step in pbar:
|
|
# jax.set_mesh: jax>=0.11 原生 mesh 上下文 (openpi 的 sharding.set_mesh 是
|
|
# 私有 contextvar, jax 侧不可见), 增广 vmap 的 sharding constraint 依赖它
|
|
with sharding.set_mesh(mesh), jax.set_mesh(mesh):
|
|
train_state, info = ptrain_step(train_rng, train_state, batch)
|
|
infos.append(info)
|
|
if step % config.log_interval == 0:
|
|
stacked_infos = common_utils.stack_forest(infos)
|
|
reduced_info = jax.device_get(jax.tree.map(jnp.mean, stacked_infos))
|
|
info_str = ", ".join(f"{k}={v:.4f}" for k, v in reduced_info.items())
|
|
pbar.write(f"Step {step}: {info_str}")
|
|
_emit_progress("train", step=step, maxStep=config.num_train_steps, loss=float(reduced_info["loss"]))
|
|
infos = []
|
|
batch = next(data_iter)
|
|
|
|
if (step % config.save_interval == 0 and step > start_step) or step == config.num_train_steps - 1:
|
|
_checkpoints.save_state(checkpoint_manager, train_state, data_loader, step)
|
|
|
|
logging.info("Waiting for checkpoint manager to finish")
|
|
checkpoint_manager.wait_until_finished()
|
|
|
|
# 改动点 5: model_structure.json (契约快照) 写进最终 checkpoint 目录
|
|
final_dir = config.checkpoint_dir / str(config.num_train_steps)
|
|
if not final_dir.is_dir():
|
|
steps = sorted((p.name for p in config.checkpoint_dir.iterdir() if p.name.isdigit()), key=int)
|
|
final_dir = config.checkpoint_dir / steps[-1] if steps else config.checkpoint_dir
|
|
structure_path = final_dir / "model_structure.json"
|
|
structure_path.write_text(contract.to_json(), encoding="utf-8")
|
|
logging.info(f"[model_structure] written: {structure_path}")
|
|
|
|
# PI_CKPT_LOCAL: 本地盘中转的 checkpoint 在结构文件写完后整体同步回产物目录
|
|
ckpt_artifact_dir = config.checkpoint_dir
|
|
if not config.checkpoint_dir.is_relative_to(output_dir):
|
|
ckpt_artifact_dir = output_dir / "checkpoints"
|
|
logging.info(f"[ckpt-local] syncing {config.checkpoint_dir} -> {ckpt_artifact_dir}")
|
|
shutil.copytree(config.checkpoint_dir, ckpt_artifact_dir, dirs_exist_ok=True)
|
|
structure_path = ckpt_artifact_dir / final_dir.name / "model_structure.json"
|
|
if not structure_path.exists() and final_dir.name.isdigit():
|
|
for p in ckpt_artifact_dir.rglob("model_structure.json"):
|
|
structure_path = p
|
|
break
|
|
|
|
report_utils.write_artifacts(
|
|
output_dir,
|
|
[
|
|
{"name": "checkpoint", "type": "file", "portId": "checkpoint", "path": str(ckpt_artifact_dir)},
|
|
{"name": "train_report", "type": "json", "portId": "train_report",
|
|
"path": str(_write_train_report(output_dir, config, contract, ckpt_artifact_dir / final_dir.name))},
|
|
],
|
|
)
|
|
|
|
|
|
def _write_train_report(output_dir: Path, config: _config.TrainConfig, contract: _contract.Contract, final_dir: Path):
|
|
report = {
|
|
"final_step": config.num_train_steps,
|
|
"checkpoint_dir": str(final_dir),
|
|
"contract": contract.to_dict(),
|
|
"derived": contract.to_model_structure()["derived"],
|
|
"batch_size": config.batch_size,
|
|
"seed": config.seed,
|
|
}
|
|
path = output_dir / "train_report.json"
|
|
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def run() -> None:
|
|
logging.basicConfig(level=logging.INFO, force=True, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
|
|
if not output_dir:
|
|
sys.exit("[hyperparam] OUTPUT_DIR not set")
|
|
|
|
contract = _contract.Contract.from_env()
|
|
dataset_paths = resolve_dataset_paths()
|
|
|
|
config = build_train_config(contract, dataset_paths, output_dir)
|
|
# LeRobot v3 本地读取: HF_LEROBOT_HOME 指向数据集公共父目录 (多数据集共父,
|
|
# 已由 resolve_dataset_paths 校验), repo_id = 目录名。
|
|
# lerobot 的该常量在 import 时固化 (utils/constants.py:67) 且 lerobot_dataset.py
|
|
# 以 from-import 持有副本, 环境变量 setdefault 对已加载模块无效, 需双处 patch;
|
|
# 不 patch 时 root 落到 ~/.cache 导致 load_metadata FileNotFoundError -> 触发
|
|
# 联网 get_safe_version 挂死。
|
|
lerobot_home = dataset_paths[0].parent
|
|
os.environ["HF_LEROBOT_HOME"] = str(lerobot_home)
|
|
import lerobot.datasets.lerobot_dataset as _ld
|
|
import lerobot.utils.constants as _lc
|
|
|
|
_lc.HF_LEROBOT_HOME = lerobot_home
|
|
_ld.HF_LEROBOT_HOME = lerobot_home
|
|
|
|
compute_norm_stats_if_missing(config, contract)
|
|
main(config, contract, output_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|