390 lines
16 KiB
Python
390 lines
16 KiB
Python
"""三层接口契约 (design_v2 §4.5, D6/D15/D16-D19)。
|
|
|
|
一份 JSON 契约, 训练配置持有, 并序列化为 checkpoint 内 model_structure.json:
|
|
|
|
第一层: 用户能力声明 (相机槽 cam1..camN / state 注入 / 语言 / action)
|
|
第二层: 派生规格 (代码从第一层算出, 不手填): 每路 token 数 / prefix token 数
|
|
第三层: 键映射 key_map (外部数据集键 -> 内部槽位; 图像映射值为列表, 支持一对多)
|
|
|
|
内部槽名与官方语义名 (base_0_rgb 等) 脱钩 (D15): 槽名是数据字典键而非权重键
|
|
(SigLIP 权重全共享), 对官方权重 resume 零影响; 槽数 N 理论无上限 (实际上限
|
|
来自显存/序列长度, 契约校验对超大 N 打 WARN 不阻断); cam 序号即 prefix token
|
|
拼接顺序, 契约随 checkpoint 锁定, 推理必须同序。
|
|
|
|
图像输入固定 1:1 (224x224, 16x16 网格, 256 token/路): 非 1:1 比例 (4:3/16:9
|
|
及 pos_emb 行切片) 已按决议移除, 模型代码还原为官方状态; 契约中出现的
|
|
aspect_ratio 字段仅接受 "1:1" (旧契约兼容), 其余取值非 0 退出。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger("openpi.contract")
|
|
|
|
# 1:1 固定规格: 224x224, patch 14, 16x16 网格 (与官方 model.IMAGE_RESOLUTION 一致)
|
|
TOKENS_PER_CAM = 256
|
|
|
|
NORM_MODES = ("min_max", "mean_std", "vision_only", "quantile")
|
|
|
|
_SLOT_RE = re.compile(r"^cam(\d+)$")
|
|
# 超大 N 只 WARN 不阻断 (D15: 实际上限来自显存/序列长度)
|
|
_WARN_CAMERA_COUNT = 8
|
|
|
|
_ROBOTWIN_TASK_KEY = "task"
|
|
|
|
|
|
def _fail(msg: str) -> None:
|
|
"""契约违例: 解析时强制, 非 0 退出且报错可读 (design_v2 §4.5)。"""
|
|
sys.exit(f"[contract] {msg}")
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Contract:
|
|
"""解析后的契约。第二层派生量经 property 计算, 不落盘不手填。"""
|
|
|
|
# 第一层: 用户能力声明
|
|
cameras: tuple[str, ...]
|
|
state_dim: int
|
|
state_inject: bool
|
|
language_enabled: bool
|
|
max_token_len: int
|
|
action_dim: int
|
|
action_horizon: int
|
|
norm_mode: str
|
|
# 第三层: 键映射 (外部键 -> 内部槽位)
|
|
key_map: dict[str, Any]
|
|
|
|
# ------------------------------------------------------------------
|
|
# 解析与校验
|
|
# ------------------------------------------------------------------
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: dict) -> "Contract":
|
|
if not isinstance(data, dict):
|
|
_fail(f"contract must be a JSON object, got {type(data).__name__}")
|
|
|
|
cameras = data.get("cameras")
|
|
if not isinstance(cameras, list) or not cameras:
|
|
_fail("'cameras' must be a non-empty list")
|
|
for cam in cameras:
|
|
if not isinstance(cam, str) or not _SLOT_RE.match(cam):
|
|
_fail(f"camera slot name must match cam<N>, got {cam!r}")
|
|
if len(set(cameras)) != len(cameras):
|
|
_fail(f"duplicate camera slots: {cameras}")
|
|
cameras = tuple(cameras)
|
|
if len(cameras) >= _WARN_CAMERA_COUNT:
|
|
logger.warning(
|
|
"contract declares %d cameras; prefix ~= %d image tokens + language, "
|
|
"memory/attention cost grows superlinearly (accepted, WARN only)",
|
|
len(cameras),
|
|
len(cameras) * 256,
|
|
)
|
|
|
|
aspect = data.get("aspect_ratio", "1:1")
|
|
if aspect != "1:1":
|
|
_fail(
|
|
f"aspect_ratio {aspect!r} is no longer supported: "
|
|
"image input is fixed to 1:1 (224x224); the non-1:1 aspect feature "
|
|
"and its model modifications have been removed"
|
|
)
|
|
|
|
state = data.get("state", {})
|
|
if not isinstance(state, dict):
|
|
_fail("'state' must be an object {dim, inject}")
|
|
state_dim = state.get("dim")
|
|
if not isinstance(state_dim, int) or state_dim <= 0:
|
|
_fail(f"'state.dim' must be a positive int, got {state_dim!r}")
|
|
state_inject = state.get("inject", True)
|
|
if not isinstance(state_inject, bool):
|
|
_fail(f"'state.inject' must be bool, got {state_inject!r}")
|
|
|
|
language = data.get("language", {})
|
|
if not isinstance(language, dict):
|
|
_fail("'language' must be an object {enabled, max_token_len}")
|
|
language_enabled = language.get("enabled", True)
|
|
if not isinstance(language_enabled, bool):
|
|
_fail(f"'language.enabled' must be bool, got {language_enabled!r}")
|
|
max_token_len = language.get("max_token_len", 200)
|
|
if not isinstance(max_token_len, int) or max_token_len <= 0:
|
|
_fail(f"'language.max_token_len' must be a positive int, got {max_token_len!r}")
|
|
|
|
action = data.get("action", {})
|
|
if not isinstance(action, dict):
|
|
_fail("'action' must be an object {dim, horizon}")
|
|
action_dim = action.get("dim")
|
|
if not isinstance(action_dim, int) or action_dim <= 0:
|
|
_fail(f"'action.dim' must be a positive int, got {action_dim!r}")
|
|
action_horizon = action.get("horizon")
|
|
if not isinstance(action_horizon, int) or action_horizon <= 0:
|
|
_fail(f"'action.horizon' must be a positive int, got {action_horizon!r}")
|
|
|
|
norm_mode = data.get("norm_mode", "min_max")
|
|
if norm_mode not in NORM_MODES:
|
|
_fail(f"norm_mode must be one of {NORM_MODES}, got {norm_mode!r}")
|
|
if norm_mode == "mean_std" and state_inject:
|
|
logger.warning(
|
|
"mean_std + state inject: z-score unbounded, clipped to q01/q99 "
|
|
"before discretization (D19, aligned with author quantile intent)"
|
|
)
|
|
if norm_mode == "vision_only" and state_inject:
|
|
logger.info("vision_only: state not injected into prompt; prefix deviates "
|
|
"from pi05_base pretraining distribution, needs enough finetune steps")
|
|
|
|
key_map = data.get("key_map")
|
|
if not isinstance(key_map, dict) or not key_map:
|
|
_fail("'key_map' must be a non-empty object")
|
|
|
|
contract = cls(
|
|
cameras=cameras,
|
|
state_dim=state_dim,
|
|
state_inject=state_inject,
|
|
language_enabled=language_enabled,
|
|
max_token_len=max_token_len,
|
|
action_dim=action_dim,
|
|
action_horizon=action_horizon,
|
|
norm_mode=norm_mode,
|
|
key_map=dict(key_map),
|
|
)
|
|
contract._validate_key_map()
|
|
return contract
|
|
|
|
@classmethod
|
|
def from_json(cls, text: str) -> "Contract":
|
|
try:
|
|
data = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
_fail(f"CONTRACT_JSON is not valid JSON: {exc}")
|
|
return cls.from_dict(data)
|
|
|
|
@classmethod
|
|
def from_env(cls, env_var: str = "CONTRACT_JSON") -> "Contract":
|
|
import os
|
|
|
|
raw = os.environ.get(env_var, "").strip()
|
|
if not raw:
|
|
return cls.robotwin_default()
|
|
return cls.from_json(raw)
|
|
|
|
@classmethod
|
|
def from_model_structure(cls, path: str) -> "Contract":
|
|
"""从 checkpoint 内 model_structure.json 恢复契约 (serve/convert/量化消费)。"""
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
except OSError as exc:
|
|
_fail(f"cannot read model_structure.json: {exc}")
|
|
if "contract" not in data:
|
|
_fail(f"model_structure.json at {path} has no 'contract' field")
|
|
return cls.from_dict(data["contract"])
|
|
|
|
# ------------------------------------------------------------------
|
|
# key_map 校验 (D15 规则, 解析时强制)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _validate_key_map(self) -> None:
|
|
single_targets: dict[str, str] = {}
|
|
slot_sources: dict[str, list[str]] = {}
|
|
for source, target in self.key_map.items():
|
|
if not isinstance(source, str) or not source:
|
|
_fail(f"key_map source must be a non-empty string, got {source!r}")
|
|
|
|
if source in _RESERVED_TARGET_KEYS:
|
|
continue # state/action/prompt 单目标, 在下面统一断言
|
|
|
|
if isinstance(target, list):
|
|
# 图像映射: 值必须是列表 (一对多允许, 同图复用)
|
|
for slot in target:
|
|
if not isinstance(slot, str):
|
|
_fail(f"key_map['{source}'] target list must contain strings, got {slot!r}")
|
|
if slot not in self.cameras:
|
|
_fail(
|
|
f"key_map['{source}'] targets unknown slot {slot!r}; "
|
|
f"declared cameras: {list(self.cameras)}"
|
|
)
|
|
slot_sources.setdefault(slot, []).append(source)
|
|
continue
|
|
|
|
if isinstance(target, str):
|
|
if target in self.cameras:
|
|
_fail(
|
|
f"key_map['{source}'] -> {target!r}: image mapping value must be a LIST "
|
|
f"(one-to-many supported), got bare string"
|
|
)
|
|
single_targets[source] = target
|
|
continue
|
|
|
|
_fail(f"key_map['{source}'] target must be a string or list, got {target!r}")
|
|
|
|
# 每个声明的 cam 必须恰好被映射一次; 多源 -> 同槽为冲突
|
|
for slot in self.cameras:
|
|
sources = slot_sources.get(slot, [])
|
|
if len(sources) == 0:
|
|
_fail(f"camera slot {slot!r} declared but not mapped in key_map")
|
|
if len(sources) > 1:
|
|
_fail(f"camera slot {slot!r} mapped by multiple sources {sources} (conflict)")
|
|
|
|
# state/action/prompt 单目标映射
|
|
for need, keys in (
|
|
("state", ("state/concat", "state", "observation.state")),
|
|
("action", ("action/concat", "action", "actions")),
|
|
("prompt", ("prompt", "task", "instructions")),
|
|
):
|
|
hits = [k for k in self.key_map if k in keys]
|
|
if len(hits) == 0:
|
|
_fail(f"key_map is missing a {need} mapping (expected one of {keys})")
|
|
if len(hits) > 1:
|
|
_fail(f"key_map has multiple {need} mappings: {hits} (single target only)")
|
|
|
|
# ------------------------------------------------------------------
|
|
# 第二层: 派生规格 (property 计算, 不手填)
|
|
# ------------------------------------------------------------------
|
|
|
|
@property
|
|
def per_cam_resolution(self) -> tuple[int, int]:
|
|
"""固定 1:1: (224, 224)。保留 property 以兼容消费方调用点。"""
|
|
return (224, 224)
|
|
|
|
@property
|
|
def tokens_per_cam(self) -> int:
|
|
"""固定 1:1: 16x16 网格 256 token。"""
|
|
return TOKENS_PER_CAM
|
|
|
|
@property
|
|
def num_cameras(self) -> int:
|
|
return len(self.cameras)
|
|
|
|
@property
|
|
def prefix_tokens(self) -> int:
|
|
return self.num_cameras * self.tokens_per_cam + self.max_token_len
|
|
|
|
# ------------------------------------------------------------------
|
|
# 键映射视图 (消费方: repack / serve 逆映射 / 量化校验)
|
|
# ------------------------------------------------------------------
|
|
|
|
@property
|
|
def vision_map(self) -> dict[str, tuple[str, ...]]:
|
|
"""外部图像键 -> 槽位元组 (一对多)。"""
|
|
return {
|
|
src: tuple(target)
|
|
for src, target in self.key_map.items()
|
|
if isinstance(target, list)
|
|
}
|
|
|
|
@property
|
|
def state_source(self) -> str:
|
|
hits = [k for k in ("state/concat", "state", "observation.state") if k in self.key_map]
|
|
return self.key_map[hits[0]] if hits else "state"
|
|
|
|
@property
|
|
def action_source(self) -> str:
|
|
hits = [k for k in ("action/concat", "action", "actions") if k in self.key_map]
|
|
return self.key_map[hits[0]] if hits else "actions"
|
|
|
|
@property
|
|
def prompt_source(self) -> str:
|
|
hits = [k for k in ("prompt", "task", "instructions") if k in self.key_map]
|
|
return self.key_map[hits[0]] if hits else "prompt"
|
|
|
|
def inverse_vision_map(self) -> dict[str, str]:
|
|
"""槽位 -> 外部键 (serve 侧: 评测 obs 对外键 -> 内部槽)。"""
|
|
inverse = {}
|
|
for source, slots in self.vision_map.items():
|
|
for slot in slots:
|
|
inverse[slot] = source
|
|
return inverse
|
|
|
|
# ------------------------------------------------------------------
|
|
# 模型配置参数 (Pi0Config 契约化字段)
|
|
# ------------------------------------------------------------------
|
|
|
|
@property
|
|
def discrete_state_input(self) -> bool:
|
|
if self.norm_mode == "vision_only":
|
|
return False
|
|
return self.state_inject
|
|
|
|
@property
|
|
def use_quantile_norm(self) -> bool:
|
|
return self.norm_mode == "quantile"
|
|
|
|
def pi0_config_kwargs(self) -> dict[str, Any]:
|
|
"""Pi0Config 契约化字段 (image_keys + 归一化联动; 分辨率固定 224x224 官方值)。"""
|
|
return {
|
|
"image_keys": tuple(self.cameras),
|
|
"action_dim": self.action_dim,
|
|
"action_horizon": self.action_horizon,
|
|
"max_token_len": self.max_token_len,
|
|
"pi05": True,
|
|
"discrete_state_input": self.discrete_state_input,
|
|
}
|
|
|
|
# ------------------------------------------------------------------
|
|
# 序列化 (checkpoint 内 model_structure.json 锚定, D13/R-9)
|
|
# ------------------------------------------------------------------
|
|
|
|
def to_model_structure(self) -> dict:
|
|
"""model_structure.json 内容: 契约全文 + 派生规格快照 (浮点结构变更的量化锚点)。"""
|
|
return {
|
|
"contract": self.to_dict(),
|
|
"derived": {
|
|
"per_cam_resolution": list(self.per_cam_resolution),
|
|
"tokens_per_cam": self.tokens_per_cam,
|
|
"prefix_tokens": self.prefix_tokens,
|
|
"num_cameras": self.num_cameras,
|
|
"discrete_state_input": self.discrete_state_input,
|
|
"use_quantile_norm": self.use_quantile_norm,
|
|
},
|
|
}
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"cameras": list(self.cameras),
|
|
"state": {"dim": self.state_dim, "inject": self.state_inject},
|
|
"language": {"enabled": self.language_enabled, "max_token_len": self.max_token_len},
|
|
"action": {"dim": self.action_dim, "horizon": self.action_horizon},
|
|
"norm_mode": self.norm_mode,
|
|
"key_map": dict(self.key_map),
|
|
}
|
|
|
|
def to_json(self) -> str:
|
|
return json.dumps(self.to_model_structure(), ensure_ascii=False, indent=2)
|
|
|
|
# ------------------------------------------------------------------
|
|
# 出厂默认 (RoboTwin 四相机 -> cam1..cam4)
|
|
# ------------------------------------------------------------------
|
|
|
|
@classmethod
|
|
def robotwin_default(cls) -> "Contract":
|
|
"""RoboTwin 数据集默认契约: 四相机 1:1, 14 维 state/action, min_max 归一化。
|
|
|
|
外部键 = data2lerobot 转换产物键 (observation.images.* / observation.state /
|
|
action / task)。
|
|
"""
|
|
return cls.from_dict(
|
|
{
|
|
"cameras": ["cam1", "cam2", "cam3", "cam4"],
|
|
"state": {"dim": 14, "inject": True},
|
|
"language": {"enabled": True, "max_token_len": 200},
|
|
"action": {"dim": 14, "horizon": 10},
|
|
"norm_mode": "min_max",
|
|
"key_map": {
|
|
"observation.images.cam_high": ["cam1"],
|
|
"observation.images.cam_third_view": ["cam2"],
|
|
"observation.images.cam_left_wrist": ["cam3"],
|
|
"observation.images.cam_right_wrist": ["cam4"],
|
|
"observation.state": "state",
|
|
"action": "actions",
|
|
_ROBOTWIN_TASK_KEY: "prompt",
|
|
},
|
|
}
|
|
)
|
|
|
|
|
|
_RESERVED_TARGET_KEYS = ("observation.state", "action", "actions", "prompt", "task", "instructions")
|