2026-10-09 16:41:13 +08:00

390 lines
16 KiB
Python

"""三层接口契约 (design_v2 §4.5, D6/D15/D16-D19)。
一份 JSON 契约, 训练配置持有, 并序列化为 checkpoint 内 model_structure.json:
第一层: 用户能力声明 (相机槽 cam1..camN / state 注入 / 语言 / action)
第二层: 派生规格 (代码从第一层算出, 不手填): 每路 token 数 / prefix token 数
第三层: 键映射 key_map (外部数据集键 -> 内部槽位; 图像映射值为列表, 支持一对多)
内部槽名与官方语义名 (base_0_rgb 等) 脱钩 (D15): 槽名是数据字典键而非权重键
(SigLIP 权重全共享), 对官方权重 resume 零影响; 槽数 N 理论无上限 (实际上限
来自显存/序列长度, 契约校验对超大 N 打 WARN 不阻断); cam 序号即 prefix token
拼接顺序, 契约随 checkpoint 锁定, 推理必须同序。
图像输入固定 1:1 (224x224, 16x16 网格, 256 token/路): 非 1:1 比例 (4:3/16:9
及 pos_emb 行切片) 已按决议移除, 模型代码还原为官方状态; 契约中出现的
aspect_ratio 字段仅接受 "1:1" (旧契约兼容), 其余取值非 0 退出。
"""
from __future__ import annotations
import dataclasses
import json
import logging
import re
import sys
from typing import Any
logger = logging.getLogger("openpi.contract")
# 1:1 固定规格: 224x224, patch 14, 16x16 网格 (与官方 model.IMAGE_RESOLUTION 一致)
TOKENS_PER_CAM = 256
NORM_MODES = ("min_max", "mean_std", "vision_only", "quantile")
_SLOT_RE = re.compile(r"^cam(\d+)$")
# 超大 N 只 WARN 不阻断 (D15: 实际上限来自显存/序列长度)
_WARN_CAMERA_COUNT = 8
_ROBOTWIN_TASK_KEY = "task"
def _fail(msg: str) -> None:
"""契约违例: 解析时强制, 非 0 退出且报错可读 (design_v2 §4.5)。"""
sys.exit(f"[contract] {msg}")
@dataclasses.dataclass(frozen=True)
class Contract:
"""解析后的契约。第二层派生量经 property 计算, 不落盘不手填。"""
# 第一层: 用户能力声明
cameras: tuple[str, ...]
state_dim: int
state_inject: bool
language_enabled: bool
max_token_len: int
action_dim: int
action_horizon: int
norm_mode: str
# 第三层: 键映射 (外部键 -> 内部槽位)
key_map: dict[str, Any]
# ------------------------------------------------------------------
# 解析与校验
# ------------------------------------------------------------------
@classmethod
def from_dict(cls, data: dict) -> "Contract":
if not isinstance(data, dict):
_fail(f"contract must be a JSON object, got {type(data).__name__}")
cameras = data.get("cameras")
if not isinstance(cameras, list) or not cameras:
_fail("'cameras' must be a non-empty list")
for cam in cameras:
if not isinstance(cam, str) or not _SLOT_RE.match(cam):
_fail(f"camera slot name must match cam<N>, got {cam!r}")
if len(set(cameras)) != len(cameras):
_fail(f"duplicate camera slots: {cameras}")
cameras = tuple(cameras)
if len(cameras) >= _WARN_CAMERA_COUNT:
logger.warning(
"contract declares %d cameras; prefix ~= %d image tokens + language, "
"memory/attention cost grows superlinearly (accepted, WARN only)",
len(cameras),
len(cameras) * 256,
)
aspect = data.get("aspect_ratio", "1:1")
if aspect != "1:1":
_fail(
f"aspect_ratio {aspect!r} is no longer supported: "
"image input is fixed to 1:1 (224x224); the non-1:1 aspect feature "
"and its model modifications have been removed"
)
state = data.get("state", {})
if not isinstance(state, dict):
_fail("'state' must be an object {dim, inject}")
state_dim = state.get("dim")
if not isinstance(state_dim, int) or state_dim <= 0:
_fail(f"'state.dim' must be a positive int, got {state_dim!r}")
state_inject = state.get("inject", True)
if not isinstance(state_inject, bool):
_fail(f"'state.inject' must be bool, got {state_inject!r}")
language = data.get("language", {})
if not isinstance(language, dict):
_fail("'language' must be an object {enabled, max_token_len}")
language_enabled = language.get("enabled", True)
if not isinstance(language_enabled, bool):
_fail(f"'language.enabled' must be bool, got {language_enabled!r}")
max_token_len = language.get("max_token_len", 200)
if not isinstance(max_token_len, int) or max_token_len <= 0:
_fail(f"'language.max_token_len' must be a positive int, got {max_token_len!r}")
action = data.get("action", {})
if not isinstance(action, dict):
_fail("'action' must be an object {dim, horizon}")
action_dim = action.get("dim")
if not isinstance(action_dim, int) or action_dim <= 0:
_fail(f"'action.dim' must be a positive int, got {action_dim!r}")
action_horizon = action.get("horizon")
if not isinstance(action_horizon, int) or action_horizon <= 0:
_fail(f"'action.horizon' must be a positive int, got {action_horizon!r}")
norm_mode = data.get("norm_mode", "min_max")
if norm_mode not in NORM_MODES:
_fail(f"norm_mode must be one of {NORM_MODES}, got {norm_mode!r}")
if norm_mode == "mean_std" and state_inject:
logger.warning(
"mean_std + state inject: z-score unbounded, clipped to q01/q99 "
"before discretization (D19, aligned with author quantile intent)"
)
if norm_mode == "vision_only" and state_inject:
logger.info("vision_only: state not injected into prompt; prefix deviates "
"from pi05_base pretraining distribution, needs enough finetune steps")
key_map = data.get("key_map")
if not isinstance(key_map, dict) or not key_map:
_fail("'key_map' must be a non-empty object")
contract = cls(
cameras=cameras,
state_dim=state_dim,
state_inject=state_inject,
language_enabled=language_enabled,
max_token_len=max_token_len,
action_dim=action_dim,
action_horizon=action_horizon,
norm_mode=norm_mode,
key_map=dict(key_map),
)
contract._validate_key_map()
return contract
@classmethod
def from_json(cls, text: str) -> "Contract":
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
_fail(f"CONTRACT_JSON is not valid JSON: {exc}")
return cls.from_dict(data)
@classmethod
def from_env(cls, env_var: str = "CONTRACT_JSON") -> "Contract":
import os
raw = os.environ.get(env_var, "").strip()
if not raw:
return cls.robotwin_default()
return cls.from_json(raw)
@classmethod
def from_model_structure(cls, path: str) -> "Contract":
"""从 checkpoint 内 model_structure.json 恢复契约 (serve/convert/量化消费)。"""
try:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
except OSError as exc:
_fail(f"cannot read model_structure.json: {exc}")
if "contract" not in data:
_fail(f"model_structure.json at {path} has no 'contract' field")
return cls.from_dict(data["contract"])
# ------------------------------------------------------------------
# key_map 校验 (D15 规则, 解析时强制)
# ------------------------------------------------------------------
def _validate_key_map(self) -> None:
single_targets: dict[str, str] = {}
slot_sources: dict[str, list[str]] = {}
for source, target in self.key_map.items():
if not isinstance(source, str) or not source:
_fail(f"key_map source must be a non-empty string, got {source!r}")
if source in _RESERVED_TARGET_KEYS:
continue # state/action/prompt 单目标, 在下面统一断言
if isinstance(target, list):
# 图像映射: 值必须是列表 (一对多允许, 同图复用)
for slot in target:
if not isinstance(slot, str):
_fail(f"key_map['{source}'] target list must contain strings, got {slot!r}")
if slot not in self.cameras:
_fail(
f"key_map['{source}'] targets unknown slot {slot!r}; "
f"declared cameras: {list(self.cameras)}"
)
slot_sources.setdefault(slot, []).append(source)
continue
if isinstance(target, str):
if target in self.cameras:
_fail(
f"key_map['{source}'] -> {target!r}: image mapping value must be a LIST "
f"(one-to-many supported), got bare string"
)
single_targets[source] = target
continue
_fail(f"key_map['{source}'] target must be a string or list, got {target!r}")
# 每个声明的 cam 必须恰好被映射一次; 多源 -> 同槽为冲突
for slot in self.cameras:
sources = slot_sources.get(slot, [])
if len(sources) == 0:
_fail(f"camera slot {slot!r} declared but not mapped in key_map")
if len(sources) > 1:
_fail(f"camera slot {slot!r} mapped by multiple sources {sources} (conflict)")
# state/action/prompt 单目标映射
for need, keys in (
("state", ("state/concat", "state", "observation.state")),
("action", ("action/concat", "action", "actions")),
("prompt", ("prompt", "task", "instructions")),
):
hits = [k for k in self.key_map if k in keys]
if len(hits) == 0:
_fail(f"key_map is missing a {need} mapping (expected one of {keys})")
if len(hits) > 1:
_fail(f"key_map has multiple {need} mappings: {hits} (single target only)")
# ------------------------------------------------------------------
# 第二层: 派生规格 (property 计算, 不手填)
# ------------------------------------------------------------------
@property
def per_cam_resolution(self) -> tuple[int, int]:
"""固定 1:1: (224, 224)。保留 property 以兼容消费方调用点。"""
return (224, 224)
@property
def tokens_per_cam(self) -> int:
"""固定 1:1: 16x16 网格 256 token。"""
return TOKENS_PER_CAM
@property
def num_cameras(self) -> int:
return len(self.cameras)
@property
def prefix_tokens(self) -> int:
return self.num_cameras * self.tokens_per_cam + self.max_token_len
# ------------------------------------------------------------------
# 键映射视图 (消费方: repack / serve 逆映射 / 量化校验)
# ------------------------------------------------------------------
@property
def vision_map(self) -> dict[str, tuple[str, ...]]:
"""外部图像键 -> 槽位元组 (一对多)。"""
return {
src: tuple(target)
for src, target in self.key_map.items()
if isinstance(target, list)
}
@property
def state_source(self) -> str:
hits = [k for k in ("state/concat", "state", "observation.state") if k in self.key_map]
return self.key_map[hits[0]] if hits else "state"
@property
def action_source(self) -> str:
hits = [k for k in ("action/concat", "action", "actions") if k in self.key_map]
return self.key_map[hits[0]] if hits else "actions"
@property
def prompt_source(self) -> str:
hits = [k for k in ("prompt", "task", "instructions") if k in self.key_map]
return self.key_map[hits[0]] if hits else "prompt"
def inverse_vision_map(self) -> dict[str, str]:
"""槽位 -> 外部键 (serve 侧: 评测 obs 对外键 -> 内部槽)。"""
inverse = {}
for source, slots in self.vision_map.items():
for slot in slots:
inverse[slot] = source
return inverse
# ------------------------------------------------------------------
# 模型配置参数 (Pi0Config 契约化字段)
# ------------------------------------------------------------------
@property
def discrete_state_input(self) -> bool:
if self.norm_mode == "vision_only":
return False
return self.state_inject
@property
def use_quantile_norm(self) -> bool:
return self.norm_mode == "quantile"
def pi0_config_kwargs(self) -> dict[str, Any]:
"""Pi0Config 契约化字段 (image_keys + 归一化联动; 分辨率固定 224x224 官方值)。"""
return {
"image_keys": tuple(self.cameras),
"action_dim": self.action_dim,
"action_horizon": self.action_horizon,
"max_token_len": self.max_token_len,
"pi05": True,
"discrete_state_input": self.discrete_state_input,
}
# ------------------------------------------------------------------
# 序列化 (checkpoint 内 model_structure.json 锚定, D13/R-9)
# ------------------------------------------------------------------
def to_model_structure(self) -> dict:
"""model_structure.json 内容: 契约全文 + 派生规格快照 (浮点结构变更的量化锚点)。"""
return {
"contract": self.to_dict(),
"derived": {
"per_cam_resolution": list(self.per_cam_resolution),
"tokens_per_cam": self.tokens_per_cam,
"prefix_tokens": self.prefix_tokens,
"num_cameras": self.num_cameras,
"discrete_state_input": self.discrete_state_input,
"use_quantile_norm": self.use_quantile_norm,
},
}
def to_dict(self) -> dict:
return {
"cameras": list(self.cameras),
"state": {"dim": self.state_dim, "inject": self.state_inject},
"language": {"enabled": self.language_enabled, "max_token_len": self.max_token_len},
"action": {"dim": self.action_dim, "horizon": self.action_horizon},
"norm_mode": self.norm_mode,
"key_map": dict(self.key_map),
}
def to_json(self) -> str:
return json.dumps(self.to_model_structure(), ensure_ascii=False, indent=2)
# ------------------------------------------------------------------
# 出厂默认 (RoboTwin 四相机 -> cam1..cam4)
# ------------------------------------------------------------------
@classmethod
def robotwin_default(cls) -> "Contract":
"""RoboTwin 数据集默认契约: 四相机 1:1, 14 维 state/action, min_max 归一化。
外部键 = data2lerobot 转换产物键 (observation.images.* / observation.state /
action / task)。
"""
return cls.from_dict(
{
"cameras": ["cam1", "cam2", "cam3", "cam4"],
"state": {"dim": 14, "inject": True},
"language": {"enabled": True, "max_token_len": 200},
"action": {"dim": 14, "horizon": 10},
"norm_mode": "min_max",
"key_map": {
"observation.images.cam_high": ["cam1"],
"observation.images.cam_third_view": ["cam2"],
"observation.images.cam_left_wrist": ["cam3"],
"observation.images.cam_right_wrist": ["cam4"],
"observation.state": "state",
"action": "actions",
_ROBOTWIN_TASK_KEY: "prompt",
},
}
)
_RESERVED_TARGET_KEYS = ("observation.state", "action", "actions", "prompt", "task", "instructions")