OpenPi_POC_Dev/pi_pipeline/train_runner.py
2026-10-09 16:41:13 +08:00

614 lines
28 KiB
Python

# 来源: openpi 仓库 scripts/train.py @ 15a9616 (JAX 训练主循环拷贝改造)
# 改动点:
# 1) 入口从 tyro CLI 配置名改为超参环境变量 + CONTRACT_JSON 契约 (pi_pipeline.contract),
# TrainConfig 由 build_train_config() 程序化组装, 不注册进 _CONFIGS
# 2) wandb 强制关闭 (design_v2 §10.2: 离线镜像 wandb 禁用), 首批图像 wandb 日志移除
# 3) 训练循环按 log_interval 打 DROBOTICFLOW_PROGRESS (RFC3339 updatedAt)
# 4) norm_stats 缺失时自动补算 (幂等: 存在即跳过, D11); 优先级 上游 NORM_STATS_PATH 显式
# 指定 > 自动计算
# 5) checkpoint 收尾写 model_structure.json (契约快照, 量化/serve/convert 锚点, D13/R-9)
# 与 $OUTPUT_DIR/.artifacts.json
# 6) 训练入口超参: DATASET_PATH (与上游端口二选一上游优先) / CONTRACT_JSON / KEY_MAP /
# RESUME_CKPT / NORM_STATS_PATH / BATCH_SIZE / NUM_TRAIN_STEPS / LR_WARMUP_STEPS /
# LR_PEAK / LR_DECAY_STEPS / SAVE_INTERVAL / KEEP_PERIOD / SEED / NUM_WORKERS / RESUME /
# OVERWRITE / EXP_NAME
from __future__ import annotations
import dataclasses
import functools
import json
import logging
import os
import shutil
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path
import etils.epath as epath # noqa: F401 (openpi 训练组件依赖此命名空间可用)
import flax.nnx as nnx
from flax.training import common_utils
import jax
import jax.numpy as jnp
import numpy as np
import optax
import tqdm_loggable.auto as tqdm
import openpi.models.model as _model
import openpi.shared.normalize as _normalize
import openpi.shared.nnx_utils as nnx_utils
import openpi.training.checkpoints as _checkpoints
import openpi.training.config as _config
import openpi.training.data_loader as _data_loader
import openpi.training.optimizer as _optimizer
import openpi.training.sharding as sharding
import openpi.training.utils as training_utils
import openpi.training.weight_loaders as _weight_loaders
import openpi.transforms as _transforms
from openpi.models.pi0_config import Pi0Config
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pi_pipeline import contract as _contract # noqa: E402
from ws_common import input_locate, report_utils # noqa: E402
# 镜像内置官方权重 (构建期下载, D8: 只打 JAX pi05_base 一份)
OFFICIAL_PI05_BASE = "/app/src/openpi_weights/pi05_base/params"
logger = logging.getLogger("openpi")
def _env_int(key: str, default: int) -> int:
raw = os.environ.get(key, "").strip()
if not raw:
return default
try:
return int(raw)
except ValueError:
sys.exit(f"[hyperparam] {key}: invalid int value {raw!r}")
def _env_float(key: str, default: float) -> float:
raw = os.environ.get(key, "").strip()
if not raw:
return default
try:
return float(raw)
except ValueError:
sys.exit(f"[hyperparam] {key}: invalid float value {raw!r}")
# ============================================================
# 契约驱动的 DataConfigFactory
# ============================================================
class ContractDataConfigFactory(_config.DataConfigFactory):
"""把契约 key_map 编译成 repack transforms + 归一化模式的 DataConfig。
多数据集聚合: dataset_paths 为 1..N 个共父目录的数据集根; 所有数据集共用
同一套契约 transforms 与同一份 norm_stats (统计在全量混合上计算), 主
repo_id = 第一个数据集目录名, 其余进 extra_repo_ids (data_loader 侧
ConcatDataset 拼接)。asset_id 单数据集保持目录名 (向后兼容旧统计产物),
多数据集用确定性的混合 id。"""
def __init__(self, dataset_paths: list[Path], contract: _contract.Contract, assets_dir: Path):
if not dataset_paths:
raise ValueError("dataset_paths must be non-empty")
super().__init__(repo_id=dataset_paths[0].name)
self._dataset_paths = dataset_paths
self._contract = contract
self._assets_dir = assets_dir
@property
def asset_id(self) -> str:
names = [p.name for p in self._dataset_paths]
if len(names) == 1:
return names[0]
digest = __import__("hashlib").md5("|".join(names).encode()).hexdigest()[:8]
return f"mix{len(names)}-{digest}"
def create(self, assets_dirs, model_config: _model.BaseModelConfig) -> _config.DataConfig:
del assets_dirs # 数据集与统计均锚定显式路径, 不走全局 assets 目录
c = self._contract
# repack structure: 旧键扁平路径 -> 新键 (image/<slot>, state, actions);
# prompt 键透传 (PromptFromLeRobotTask 在 dataset 层已产出 "prompt",
# repack 重建 dict 时需显式保留, 否则 TokenizePrompt 报 Prompt is required)
image_struct = {slot: src for src, slots in c.vision_map.items() for slot in slots}
state_src = next(k for k in ("state/concat", "state", "observation.state") if k in c.key_map)
action_src = next(k for k in ("action/concat", "action", "actions") if k in c.key_map)
structure = {
"image": image_struct,
"state": state_src,
"actions": action_src,
"prompt": "prompt",
}
use_quantile = c.use_quantile_norm
norm_mode = None if c.norm_mode == "quantile" else c.norm_mode
return dataclasses.replace(
self.base_config or _config.DataConfig(),
repo_id=self._dataset_paths[0].name,
extra_repo_ids=tuple(p.name for p in self._dataset_paths[1:]),
asset_id=self.asset_id,
norm_stats=self._load_norm_stats(),
repack_transforms=_transforms.Group(inputs=[_transforms.RepackTransform(structure)]),
data_transforms=_transforms.Group(inputs=[_transforms.ImagesToHWC(), _transforms.BuildImageMasks()]),
# model_transforms: TokenizePrompt / ResizeImages / PadStatesAndActions
# (缺失时 jax 静默丢语言分支, torch 在 lang_embed 报 None)
model_transforms=_config.ModelTransformFactory()(model_config),
use_quantile_norm=use_quantile,
norm_mode=norm_mode,
action_sequence_keys=(action_src,),
prompt_from_task=True,
)
def dataset_configs(self, model_config: _model.BaseModelConfig) -> list[_config.DataConfig]:
"""每数据集一份 DataConfig (repo_id 各自, extra 为空, 其余共享)。
消费方: norm_stats 统计 (在全量混合上计算共享统计)。"""
base = self.create(None, model_config)
configs = [dataclasses.replace(base, repo_id=p.name, extra_repo_ids=()) for p in self._dataset_paths]
return configs
def _load_norm_stats(self):
stats_dir = self._assets_dir / self.asset_id
try:
stats = _normalize.load(stats_dir)
logging.info(f"[norm_stats] loaded from {stats_dir}")
return stats
except FileNotFoundError:
logging.info(f"[norm_stats] not found in {stats_dir} (will be computed)")
return None
# ============================================================
# TrainConfig 组装
# ============================================================
def resolve_dataset_paths() -> list[Path]:
"""输入根 (上游端口 portId=dataset 优先, DATASET_PATH 兜底) -> 数据集根列表。
根可为单数据集根或多数据集父目录 (expand_dataset_roots 展开); 多数据集
必须共父目录 (HF_LEROBOT_HOME 唯一)。每个数据集根校验 meta/info.json。
平台 workspace (PVC) 输入先整体暂存到节点本地盘再展开 (stage_to_local)。"""
root = input_locate.locate_input_dir(port_id="dataset", env_key="DATASET_PATH")
root = input_locate.stage_to_local(root)
paths = input_locate.expand_dataset_roots(root)
parents = {p.parent for p in paths}
if len(parents) != 1:
sys.exit(f"[input] datasets must share one parent dir for LeRobot local read, got {parents}")
for path in paths:
if not (path / "meta" / "info.json").is_file():
sys.exit(
f"[input] {path} is not a LeRobot v3 dataset root "
"(expected meta/info.json + data/*.parquet); got contents: "
f"{sorted(p.name for p in path.iterdir())[:10]}"
)
return paths
def build_train_config(contract: _contract.Contract, dataset_paths: list[Path], output_dir: Path) -> _config.TrainConfig:
resume_ckpt = os.environ.get("RESUME_CKPT", "").strip()
if resume_ckpt in ("", "official"):
weight_loader: _weight_loaders.WeightLoader
if os.path.isdir(OFFICIAL_PI05_BASE):
weight_loader = _weight_loaders.CheckpointWeightLoader(OFFICIAL_PI05_BASE)
else:
weight_loader = _weight_loaders.CheckpointWeightLoader("gs://openpi-assets/checkpoints/pi05_base/params")
else:
if not os.path.isdir(resume_ckpt):
sys.exit(f"[hyperparam] RESUME_CKPT directory not found: {resume_ckpt}")
weight_loader = _weight_loaders.CheckpointWeightLoader(resume_ckpt)
assets_dir = output_dir / "assets"
exp_name = os.environ.get("EXP_NAME", "").strip() or "pi05_contract"
lr_peak = _env_float("LR_PEAK", 2.5e-5)
# LORA=true: 官方 low_mem LoRA 微调惯例 (config.py pi0_libero_low_mem_finetune 同款):
# LoRA variant + freeze 非 LoRA 参数 (bf16) + 关 EMA。32GB 卡全量微调 init 峰值
# (params+EMA+AdamW 状态 replicated 中间量) 不可行时的冒烟/低显存路径。
use_lora = os.environ.get("LORA", "").strip().lower() in ("1", "true", "yes")
model_kwargs = dict(contract.pi0_config_kwargs())
if use_lora:
model_kwargs["paligemma_variant"] = "gemma_2b_lora"
model_kwargs["action_expert_variant"] = "gemma_300m_lora"
config = _config.TrainConfig(
name="pi05_contract",
project_name="datapipeline",
exp_name=exp_name,
model=Pi0Config(**model_kwargs),
weight_loader=weight_loader,
data=ContractDataConfigFactory(dataset_paths, contract, assets_dir / "pi05_contract"),
assets_base_dir=assets_dir,
# PI_CKPT_LOCAL=1: checkpoint 先写节点本地盘, 训练完成后再整体同步回
# OUTPUT_DIR/checkpoints。平台 workspace 为网络 PVC, orbax/tensorstore 的
# OCDBT manifest 原子写与 zarr .__lock 文件锁在其元数据一致性下会损坏/
# 失败; 本地直写 (本地盘部署) 保持原行为不受影响。
checkpoint_base_dir=(
Path(tempfile.gettempdir()) / "pi-ckpt" / "checkpoints"
if os.environ.get("PI_CKPT_LOCAL", "").strip().lower() in ("1", "true", "yes")
else output_dir / "checkpoints"
),
seed=_env_int("SEED", 0),
batch_size=_env_int("BATCH_SIZE", 8),
num_workers=_env_int("NUM_WORKERS", 4),
num_train_steps=_env_int("NUM_TRAIN_STEPS", 20000),
fsdp_devices=_env_int("FSDP_DEVICES", 1),
log_interval=_env_int("LOG_INTERVAL", 100),
save_interval=_env_int("SAVE_INTERVAL", 1000),
keep_period=_env_int("KEEP_PERIOD", 5000),
overwrite=os.environ.get("OVERWRITE", "").strip().lower() in ("1", "true", "yes"),
resume=os.environ.get("RESUME", "").strip().lower() in ("1", "true", "yes"),
wandb_enabled=False, # 改动点 2: 强制关闭
freeze_filter=Pi0Config(**model_kwargs).get_freeze_filter() if use_lora else nnx.Nothing(),
ema_decay=None if use_lora else _env_float("EMA_DECAY", 0.99),
lr_schedule=_optimizer.CosineDecaySchedule(
warmup_steps=_env_int("LR_WARMUP_STEPS", 1000),
peak_lr=lr_peak,
decay_steps=_env_int("LR_DECAY_STEPS", 30000),
),
optimizer=_optimizer.AdamW(),
)
logging.info(f"[contract] cameras={contract.cameras} "
f"resolution={contract.per_cam_resolution} prefix_tokens={contract.prefix_tokens} "
f"norm_mode={contract.norm_mode}")
return config
# ============================================================
# norm_stats 自动补算 (D11: 幂等, 存在即跳过)
# ============================================================
def compute_norm_stats_if_missing(config: _config.TrainConfig, contract: _contract.Contract) -> None:
explicit = os.environ.get("NORM_STATS_PATH", "").strip()
stats_dir = config.assets_dirs / config.data.asset_id
if explicit:
if not os.path.isfile(os.path.join(explicit, "norm_stats.json")):
sys.exit(f"[norm_stats] NORM_STATS_PATH has no norm_stats.json: {explicit}")
# 显式指定优先: 复制到 assets 目录供 DataConfig 加载
import shutil
shutil.copytree(explicit, stats_dir, dirs_exist_ok=True)
logging.info(f"[norm_stats] copied from explicit NORM_STATS_PATH {explicit} -> {stats_dir}")
return
if (stats_dir / "norm_stats.json").is_file():
logging.info(f"[norm_stats] exists, skip (idempotent): {stats_dir}")
return
# 多数据集: 统计在全量混合上计算 (共享一份 norm_stats, 全部数据集在
# repack+data_transform 空间逐帧累积)
logging.info(f"[norm_stats] computing in repack+data_transform space -> {stats_dir}")
stats = {"state": _normalize.RunningStats(), "actions": _normalize.RunningStats()}
total_frames = 0
for data_config in config.data.dataset_configs(config.model):
dataset = _data_loader.create_torch_dataset(data_config, config.model.action_horizon, config.model)
dataset = _data_loader.TransformedDataset(
dataset,
[
*data_config.repack_transforms.inputs,
*data_config.data_transforms.inputs,
_transforms.InjectDefaultPrompt(None),
],
)
for i in range(len(dataset)):
item = dataset[i]
for key in ("state", "actions"):
stats[key].update(np.asarray(item[key])[None, :] if key == "state" else np.asarray(item[key]))
if (total_frames + i + 1) % 500 == 0:
logging.info(f"[norm_stats] {data_config.repo_id}: {i + 1}/{len(dataset)} frames")
total_frames += len(dataset)
# 多数据集 asset 目录: norm_stats 落在共享 asset_id 下
stats_dir.parent.mkdir(parents=True, exist_ok=True)
_normalize.save(stats_dir, {k: v.get_statistics() for k, v in stats.items()})
logging.info(f"[norm_stats] saved ({total_frames}+ frames): {stats_dir}")
# ============================================================
# 训练主循环 (来源: scripts/train.py main, 改动点见文件头)
# ============================================================
def _emit_progress(stage: str, **fields) -> None:
payload = {"stage": stage, **fields}
payload["updatedAt"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
print("DROBOTICFLOW_PROGRESS " + json.dumps(payload), flush=True)
def _adapt_action_dim(params: dict, action_dim: int) -> dict:
"""D5 / design_v2 §5.2: action_dim resume 适配 — action_in/out_proj kernel 循环 tile。
官方 pi05_base 的 action_dim=32; 契约维 ad != 32 时 (ad<32 截取, ad>32 循环重复)
适配到模型结构。bias/out_features 不受影响; state 走 tokenizer 无投影权重。
"""
for proj, axis in (("action_in_proj", 0), ("action_out_proj", 1)):
node = params.get(proj)
if not isinstance(node, dict) or "kernel" not in node:
continue
kernel = np.asarray(node["kernel"])
src_dim = kernel.shape[axis]
if src_dim == action_dim:
continue
reps = -(-action_dim // src_dim)
tiled = np.tile(kernel, (reps, 1) if axis == 0 else (1, reps))
tiled = tiled[:action_dim] if axis == 0 else tiled[:, :action_dim]
node["kernel"] = tiled
logging.info(f"[resume-adapt] {proj}.kernel {kernel.shape} -> {tiled.shape} (cyclic tile, D5)")
# out_proj 的 bias 维度 = action_dim (in_proj 的 bias 是 out_features 不受影响)
if "bias" in node and np.asarray(node["bias"]).shape[:1] == (src_dim,):
bias = np.asarray(node["bias"])
node["bias"] = np.tile(bias, reps)[:action_dim]
logging.info(f"[resume-adapt] {proj}.bias {bias.shape} -> {node['bias'].shape} (cyclic tile, D5)")
return params
def _load_weights_and_validate(loader, params_shape):
loaded_params = loader.load(params_shape)
import flax.traverse_util as traverse_util
unflattened = traverse_util.unflatten_dict(
{k: v for k, v in traverse_util.flatten_dict(loaded_params).items() if not isinstance(v, jax.ShapeDtypeStruct)}
)
return unflattened
def init_train_state(config, init_rng, mesh, *, resume):
tx = _optimizer.create_optimizer(config.optimizer, config.lr_schedule, weight_decay_mask=None)
def init(rng, partial_params=None):
rng, model_rng = jax.random.split(rng)
model = config.model.create(model_rng)
if partial_params is not None:
graphdef, state = nnx.split(model)
state.replace_by_pure_dict(partial_params)
model = nnx.merge(graphdef, state)
params = nnx.state(model)
params = nnx_utils.state_map(params, config.freeze_filter, lambda p: p.replace(p.value.astype(jnp.bfloat16)))
return training_utils.TrainState(
step=0,
params=params,
model_def=nnx.graphdef(model),
tx=tx,
opt_state=tx.init(params.filter(config.trainable_filter)),
ema_decay=config.ema_decay,
ema_params=None if config.ema_decay is None else params,
)
train_state_shape = jax.eval_shape(init, init_rng)
# DataPipeline 改动: 阈值 4->32 MiB — attention qkv/out proj 等中小权重在
# jax>=0.11 下切分 (即使 out 维) 仍会在 MHA einsum 的 head 维收缩处产生
# 双边 fsdp 歧义; replicate 中小权重, 只切 embedding / gemma expert MLP 等
# 大块 (显存大头), 兼顾收缩安全与省显存
state_sharding = sharding.fsdp_sharding(train_state_shape, mesh, min_size_mbytes=32, log=True)
if resume:
return train_state_shape, state_sharding
partial_params = _load_weights_and_validate(config.weight_loader, train_state_shape.params.to_pure_dict())
partial_params = _adapt_action_dim(partial_params, config.model.action_dim)
# LoRA (freeze_filter 生效): partial 全为冻结权重, init 的 state_map 本就转 bf16;
# 提前在 jit 外转可将 replicated 输入峰值 12GB(fp32) 降至 6GB, 结果 bit 级等效,
# 为 32GB 卡的 FSDP all-gather 留头寸 (D14 单机多卡约束下的显存适配)
if not isinstance(config.freeze_filter, nnx.Nothing):
partial_params = jax.tree.map(
lambda x: x.astype(np.float32).astype(jnp.bfloat16) if np.asarray(x).dtype == np.float32 else x, partial_params
)
replicated_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
train_state = jax.jit(
init,
donate_argnums=(1,),
in_shardings=replicated_sharding,
out_shardings=state_sharding,
)(init_rng, partial_params)
return train_state, state_sharding
def train_step(config, rng, state, batch):
model = nnx.merge(state.model_def, state.params)
model.train()
observation, actions = batch
def loss_fn(model, rng, observation, actions):
chunked_loss = model.compute_loss(rng, observation, actions, train=True)
return jnp.mean(chunked_loss)
train_rng = jax.random.fold_in(rng, state.step)
observation, actions = batch
diff_state = nnx.DiffState(0, config.trainable_filter)
loss, grads = nnx.value_and_grad(loss_fn, argnums=diff_state)(model, train_rng, observation, actions)
params = state.params.filter(config.trainable_filter)
updates, new_opt_state = state.tx.update(grads, state.opt_state, params)
new_params = optax.apply_updates(params, updates)
nnx.update(model, new_params)
new_params = nnx.state(model)
new_state = dataclasses.replace(state, step=state.step + 1, params=new_params, opt_state=new_opt_state)
if state.ema_decay is not None:
new_state = dataclasses.replace(
new_state,
ema_params=jax.tree.map(
lambda old, new: state.ema_decay * old + (1 - state.ema_decay) * new, state.ema_params, new_params
),
)
kernel_params = nnx.state(
model,
nnx.All(
nnx.Param,
nnx.Not(nnx_utils.PathRegex(".*/(bias|scale|pos_embedding|input_embedding)")),
lambda _, x: x.value.ndim > 1,
),
)
info = {
"loss": loss,
"grad_norm": optax.global_norm(grads),
"param_norm": optax.global_norm(kernel_params),
}
return new_state, info
def main(config: _config.TrainConfig, contract: _contract.Contract, output_dir: Path):
logging.info(f"Running on: {jax.devices()}")
# jax>=0.11: 数据只在 batch 轴切 (数据并行度 = device_count / fsdp_devices),
# 整除性按 batch 轴校验, 不再按总设备数 (官方复合轴语义的遗留校验)
data_parallelism = jax.device_count() // max(1, config.fsdp_devices)
if config.batch_size % data_parallelism != 0:
raise ValueError(
f"Batch size {config.batch_size} must be divisible by the data parallelism {data_parallelism} "
f"(devices {jax.device_count()} / fsdp {config.fsdp_devices})."
)
rng = jax.random.key(config.seed)
train_rng, init_rng = jax.random.split(rng)
mesh = sharding.make_mesh(config.fsdp_devices)
data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.data_axis_spec(mesh)))
replicated_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
checkpoint_manager, resuming = _checkpoints.initialize_checkpoint_dir(
config.checkpoint_dir,
keep_period=config.keep_period,
overwrite=config.overwrite,
resume=config.resume,
)
data_loader = _data_loader.create_data_loader(config, sharding=data_sharding, shuffle=True)
data_iter = iter(data_loader)
batch = next(data_iter)
logging.info(f"Initialized data loader:\n{training_utils.array_tree_to_info(batch)}")
train_state, train_state_sharding = init_train_state(config, init_rng, mesh, resume=resuming)
jax.block_until_ready(train_state)
logging.info(f"Initialized train state:\n{training_utils.array_tree_to_info(train_state.params)}")
if resuming:
train_state = _checkpoints.restore_state(checkpoint_manager, train_state, data_loader)
# jax>=0.11: jit in_shardings 必须与实际数组布局一致 (dataloader 已按
# data_sharding 布局产出); 词表 gather 的歧义已由 gemma.encode 的显式
# out_sharding 解决, 不再需要 prompt replicated 特例 (单卡时代遗留)
batch_in_shardings = data_sharding
ptrain_step = jax.jit(
functools.partial(train_step, config),
in_shardings=(replicated_sharding, train_state_sharding, batch_in_shardings),
out_shardings=(train_state_sharding, replicated_sharding),
donate_argnums=(1,),
)
start_step = int(train_state.step)
pbar = tqdm.tqdm(
range(start_step, config.num_train_steps),
initial=start_step,
total=config.num_train_steps,
dynamic_ncols=True,
)
infos = []
for step in pbar:
# jax.set_mesh: jax>=0.11 原生 mesh 上下文 (openpi 的 sharding.set_mesh 是
# 私有 contextvar, jax 侧不可见), 增广 vmap 的 sharding constraint 依赖它
with sharding.set_mesh(mesh), jax.set_mesh(mesh):
train_state, info = ptrain_step(train_rng, train_state, batch)
infos.append(info)
if step % config.log_interval == 0:
stacked_infos = common_utils.stack_forest(infos)
reduced_info = jax.device_get(jax.tree.map(jnp.mean, stacked_infos))
info_str = ", ".join(f"{k}={v:.4f}" for k, v in reduced_info.items())
pbar.write(f"Step {step}: {info_str}")
_emit_progress("train", step=step, maxStep=config.num_train_steps, loss=float(reduced_info["loss"]))
infos = []
batch = next(data_iter)
if (step % config.save_interval == 0 and step > start_step) or step == config.num_train_steps - 1:
_checkpoints.save_state(checkpoint_manager, train_state, data_loader, step)
logging.info("Waiting for checkpoint manager to finish")
checkpoint_manager.wait_until_finished()
# 改动点 5: model_structure.json (契约快照) 写进最终 checkpoint 目录
final_dir = config.checkpoint_dir / str(config.num_train_steps)
if not final_dir.is_dir():
steps = sorted((p.name for p in config.checkpoint_dir.iterdir() if p.name.isdigit()), key=int)
final_dir = config.checkpoint_dir / steps[-1] if steps else config.checkpoint_dir
structure_path = final_dir / "model_structure.json"
structure_path.write_text(contract.to_json(), encoding="utf-8")
logging.info(f"[model_structure] written: {structure_path}")
# PI_CKPT_LOCAL: 本地盘中转的 checkpoint 在结构文件写完后整体同步回产物目录
ckpt_artifact_dir = config.checkpoint_dir
if not config.checkpoint_dir.is_relative_to(output_dir):
ckpt_artifact_dir = output_dir / "checkpoints"
logging.info(f"[ckpt-local] syncing {config.checkpoint_dir} -> {ckpt_artifact_dir}")
shutil.copytree(config.checkpoint_dir, ckpt_artifact_dir, dirs_exist_ok=True)
structure_path = ckpt_artifact_dir / final_dir.name / "model_structure.json"
if not structure_path.exists() and final_dir.name.isdigit():
for p in ckpt_artifact_dir.rglob("model_structure.json"):
structure_path = p
break
report_utils.write_artifacts(
output_dir,
[
{"name": "checkpoint", "type": "file", "portId": "checkpoint", "path": str(ckpt_artifact_dir)},
{"name": "train_report", "type": "json", "portId": "train_report",
"path": str(_write_train_report(output_dir, config, contract, ckpt_artifact_dir / final_dir.name))},
],
)
def _write_train_report(output_dir: Path, config: _config.TrainConfig, contract: _contract.Contract, final_dir: Path):
report = {
"final_step": config.num_train_steps,
"checkpoint_dir": str(final_dir),
"contract": contract.to_dict(),
"derived": contract.to_model_structure()["derived"],
"batch_size": config.batch_size,
"seed": config.seed,
}
path = output_dir / "train_report.json"
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def run() -> None:
logging.basicConfig(level=logging.INFO, force=True, format="%(asctime)s [%(levelname)s] %(message)s")
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
if not output_dir:
sys.exit("[hyperparam] OUTPUT_DIR not set")
contract = _contract.Contract.from_env()
dataset_paths = resolve_dataset_paths()
config = build_train_config(contract, dataset_paths, output_dir)
# LeRobot v3 本地读取: HF_LEROBOT_HOME 指向数据集公共父目录 (多数据集共父,
# 已由 resolve_dataset_paths 校验), repo_id = 目录名。
# lerobot 的该常量在 import 时固化 (utils/constants.py:67) 且 lerobot_dataset.py
# 以 from-import 持有副本, 环境变量 setdefault 对已加载模块无效, 需双处 patch;
# 不 patch 时 root 落到 ~/.cache 导致 load_metadata FileNotFoundError -> 触发
# 联网 get_safe_version 挂死。
lerobot_home = dataset_paths[0].parent
os.environ["HF_LEROBOT_HOME"] = str(lerobot_home)
import lerobot.datasets.lerobot_dataset as _ld
import lerobot.utils.constants as _lc
_lc.HF_LEROBOT_HOME = lerobot_home
_ld.HF_LEROBOT_HOME = lerobot_home
compute_norm_stats_if_missing(config, contract)
main(config, contract, output_dir)
if __name__ == "__main__":
run()