887 lines
35 KiB
Python
887 lines
35 KiB
Python
# 来源: RoboTwin仓库 scripts/eval_policy_xpolicylab.py @ 30954692d06ba7e89f7a6b76064f4062c488fa81
|
|
# 改动点:
|
|
# 1) 多任务循环由本文件承担 (TASK_NAME 逗号分隔逐任务顺序执行); 评测固定单进程
|
|
# (官方批模式开关 EVAL_BATCH 语义移除), 去多进程批调度与共享内存依赖
|
|
# 2) 相机映射改用 ws_common.camera_map (补 front_camera -> cam_third_view; 删 observer
|
|
# third_view_rgb 分支: 评测 obs 的 cam_third_view 一律来自 front_camera, 与采集对齐)
|
|
# 3) front_camera_type 注入与采集侧同源 (ws_common.config_override, 预设扩展键或超参)
|
|
# 4) STEP_LIMIT 绝对值覆盖 (空值用官方 _eval_step_limit.yml 每任务值)
|
|
# 5) SAVE_VIDEO 开关 (回灌视频写 $OUTPUT_DIR/videos/<task>/episodeN.mp4)
|
|
# 6) 启动渲染自检, 失败非 0 退出 (官方 Sapien_TEST 失败退出码为 0, 此处修正)
|
|
# 7) 逐任务结果写 $OUTPUT_DIR/eval_result/<task>/_result.txt, 汇总 eval_summary.json 写
|
|
# $OUTPUT_DIR (禁用 result.json 文件名), 结束写 .artifacts.json; 超参全部环境变量注入
|
|
# 另: legacy_tcp 分支 import 路径修正为 client_server.tcp.model_client (vendored 树实际位置)
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import traceback
|
|
from collections.abc import Mapping, Sequence
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import yaml
|
|
|
|
APP_SRC = Path(os.environ.get("APP_SRC", "/app/src"))
|
|
ROBOTWIN_ROOT = Path(os.environ.get("ROBOTWIN_ROOT", str(APP_SRC / "RoboTwin")))
|
|
PRESETS_DIR = Path(os.environ.get("PRESETS_DIR", str(APP_SRC / "presets")))
|
|
DEFAULT_XPOLICYLAB_ROOT = APP_SRC / "vendor" / "XPolicyLab"
|
|
|
|
for _p in (str(APP_SRC), str(ROBOTWIN_ROOT), str(ROBOTWIN_ROOT / "description" / "utils")):
|
|
if _p not in sys.path:
|
|
sys.path.insert(0, _p)
|
|
os.chdir(ROBOTWIN_ROOT)
|
|
|
|
from ws_common import config_override, report_utils # noqa: E402
|
|
from ws_common.camera_map import CAMERA_NAME_MAP # noqa: E402
|
|
|
|
# 配置目录合并 + CONFIGS_PATH 改写 (须先于 envs 任务类加载)
|
|
MERGED_CONFIG_DIR = config_override.build_merged_config_dir(ROBOTWIN_ROOT, PRESETS_DIR)
|
|
CONFIGS_PATH = config_override.install_config_dir(MERGED_CONFIG_DIR)
|
|
|
|
from envs.utils.create_actor import UnStableError # noqa: E402
|
|
from generate_episode_instructions import generate_episode_descriptions # noqa: E402
|
|
|
|
|
|
# ============================================================
|
|
# 渲染自检 (改动点 6: 失败非 0)
|
|
# ============================================================
|
|
|
|
def sapien_render_check() -> None:
|
|
import sapien.core as sapien # noqa: PLC0415
|
|
from sapien.render import set_global_config # noqa: PLC0415
|
|
|
|
try:
|
|
engine = sapien.Engine()
|
|
set_global_config(max_num_materials=50000, max_num_textures=50000)
|
|
renderer = sapien.SapienRenderer()
|
|
engine.set_renderer(renderer)
|
|
sapien.render.set_camera_shader_dir("rt")
|
|
sapien.render.set_ray_tracing_samples_per_pixel(32)
|
|
sapien.render.set_ray_tracing_path_depth(8)
|
|
sapien.render.set_ray_tracing_denoiser("oidn")
|
|
scene = engine.create_scene(sapien.SceneConfig())
|
|
del scene, renderer, engine
|
|
except SystemExit:
|
|
raise
|
|
except Exception as exc:
|
|
print(f"[render-check] FAILED: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
sys.exit(1)
|
|
print("[render-check] Render Well")
|
|
|
|
|
|
# ============================================================
|
|
# 超参
|
|
# ============================================================
|
|
|
|
def read_hyperparams() -> dict:
|
|
env = os.environ
|
|
hp: dict[str, Any] = {
|
|
"policy_server_ip": env.get("POLICY_SERVER_IP", ""),
|
|
"policy_server_port": env.get("POLICY_SERVER_PORT", ""),
|
|
"protocol": env.get("PROTOCOL", "") or "ws",
|
|
"policy_name": env.get("POLICY_NAME", "") or "custom_policy",
|
|
"ckpt_setting": env.get("CKPT_SETTING", "") or "default",
|
|
"task_names": [t.strip() for t in env.get("TASK_NAME", "").split(",") if t.strip()],
|
|
"task_config": env.get("TASK_CONFIG", "") or "demo_clean",
|
|
"instruction_type": env.get("INSTRUCTION_TYPE", "") or None,
|
|
"action_type": env.get("ACTION_TYPE", "") or "joint",
|
|
"step_limit": config_override.parse_typed(env.get("STEP_LIMIT", "") or "0", "int", "STEP_LIMIT"),
|
|
"action_stride": config_override.parse_typed(env.get("ACTION_STRIDE", "") or "0", "int", "ACTION_STRIDE"),
|
|
"test_num": config_override.parse_typed(env.get("TEST_NUM", "") or "5", "int", "TEST_NUM"),
|
|
"seed": config_override.parse_typed(env.get("SEED", "") or "0", "int", "SEED"),
|
|
"expert_check": config_override.parse_typed(env.get("EXPERT_CHECK", "") or "true", "bool", "EXPERT_CHECK"),
|
|
"frequency": config_override.parse_typed(env.get("FREQUENCY", "") or "30", "int", "FREQUENCY"),
|
|
"max_seed_attempts": config_override.parse_typed(env.get("MAX_SEED_ATTEMPTS", "") or "0", "int", "MAX_SEED_ATTEMPTS"),
|
|
"save_video": config_override.parse_typed(env.get("SAVE_VIDEO", "") or "true", "bool", "SAVE_VIDEO"),
|
|
"output_dir": env.get("OUTPUT_DIR", "") or os.getcwd(),
|
|
}
|
|
if not hp["policy_server_ip"] or not hp["policy_server_port"]:
|
|
sys.exit("[hyperparam] POLICY_SERVER_IP and POLICY_SERVER_PORT are required")
|
|
if not hp["task_names"]:
|
|
sys.exit("[hyperparam] TASK_NAME is required (comma-separated task list)")
|
|
if hp["protocol"] not in ("ws", "legacy_tcp"):
|
|
sys.exit(f"[hyperparam] PROTOCOL must be ws or legacy_tcp, got {hp['protocol']!r}")
|
|
if hp["action_type"] not in ("joint", "ee", "qpos", "endpose"):
|
|
sys.exit(f"[hyperparam] ACTION_TYPE must be joint or ee, got {hp['action_type']!r}")
|
|
return hp
|
|
|
|
|
|
# ============================================================
|
|
# 策略客户端 (来源: eval_policy_xpolicylab.py build_policy_client, tcp import 路径修正)
|
|
# ============================================================
|
|
|
|
def add_xpolicylab_paths(xpolicylab_root) -> None:
|
|
if not xpolicylab_root:
|
|
return
|
|
root = os.path.abspath(os.fspath(xpolicylab_root))
|
|
for path in (root, os.path.join(root, "integrations")):
|
|
if path not in sys.path:
|
|
sys.path.insert(0, path)
|
|
|
|
|
|
def build_policy_client(hp: dict):
|
|
protocol = hp["protocol"]
|
|
add_xpolicylab_paths(os.environ.get("XPOLICYLAB_ROOT", str(DEFAULT_XPOLICYLAB_ROOT)))
|
|
|
|
host = hp["policy_server_ip"]
|
|
port = int(hp["policy_server_port"])
|
|
current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S_%f")
|
|
|
|
if protocol == "ws":
|
|
from client_server.ws import WsModelClient # noqa: PLC0415
|
|
|
|
client = WsModelClient(
|
|
url=f"ws://{host}:{port}",
|
|
evaluation_id=f"robotwin-eval-{current_time}",
|
|
trial_id="robotwin-trial",
|
|
action_case_id="robotwin-case",
|
|
repeat_index=None,
|
|
)
|
|
client._robotwin_protocol = "ws"
|
|
return client
|
|
|
|
from client_server.tcp.model_client import ModelClient # noqa: PLC0415
|
|
|
|
client = ModelClient(host=host, port=port)
|
|
client._robotwin_protocol = protocol
|
|
return client
|
|
|
|
|
|
def close_policy_client(model_client) -> None:
|
|
close = getattr(model_client, "close", None)
|
|
if callable(close):
|
|
close()
|
|
|
|
|
|
# ============================================================
|
|
# 任务配置加载 (与采集侧同源: 合并目录 + 覆盖 + front 注入)
|
|
# ============================================================
|
|
|
|
def class_decorator(task_name: str):
|
|
envs_module = importlib.import_module(f"envs.{task_name}")
|
|
try:
|
|
env_class = getattr(envs_module, task_name)
|
|
return env_class()
|
|
except Exception as exc:
|
|
raise SystemExit(f"No Task: {task_name} ({exc})") from exc
|
|
|
|
|
|
def get_camera_config(camera_type: str) -> dict[str, Any]:
|
|
camera_config_path = Path(CONFIGS_PATH) / "_camera_config.yml"
|
|
with open(camera_config_path, "r", encoding="utf-8") as f:
|
|
args = yaml.safe_load(f)
|
|
if camera_type not in args:
|
|
raise KeyError(f"camera {camera_type} is not defined")
|
|
return args[camera_type]
|
|
|
|
|
|
def get_embodiment_config(robot_file: str) -> dict[str, Any]:
|
|
robot_config_file = os.path.join(robot_file, "config.yml")
|
|
with open(robot_config_file, "r", encoding="utf-8") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def ensure_xpolicylab_observation_flags(args: dict[str, Any]) -> None:
|
|
data_type = args.setdefault("data_type", {})
|
|
data_type["rgb"] = True
|
|
data_type["qpos"] = True
|
|
data_type["endpose"] = True
|
|
|
|
|
|
def resolve_eval_instruction(hp: dict[str, Any], task_args: dict[str, Any]) -> str:
|
|
instruction_type = hp.get("instruction_type")
|
|
if not instruction_type:
|
|
if "eval_instruction" not in task_args:
|
|
raise ValueError("task config must define eval_instruction as 'seen' or 'unseen'")
|
|
instruction_type = task_args["eval_instruction"]
|
|
instruction_type = str(instruction_type).strip().lower()
|
|
if instruction_type not in {"seen", "unseen"}:
|
|
raise ValueError(f"eval instruction must be 'seen' or 'unseen', got {instruction_type!r}")
|
|
return instruction_type
|
|
|
|
|
|
def load_task_args(task_name: str, hp: dict) -> dict[str, Any]:
|
|
config_path = Path(CONFIGS_PATH) / f"{hp['task_config']}.yml"
|
|
if not config_path.is_file():
|
|
available = sorted(p.stem for p in Path(CONFIGS_PATH).glob("*.yml") if not p.name.startswith("_"))
|
|
sys.exit(f"[config] task config '{hp['task_config']}' not found; available presets: {available}")
|
|
with open(config_path, "r", encoding="utf-8") as f:
|
|
args = yaml.safe_load(f)
|
|
|
|
args["task_name"] = task_name
|
|
args["task_config"] = hp["task_config"]
|
|
ensure_xpolicylab_observation_flags(args)
|
|
|
|
# 超参覆盖 (env > 预设 yml > 内置默认), 与采集侧完全相同的一组覆盖键
|
|
applied = config_override.apply_env_overrides(args)
|
|
if applied:
|
|
print("[override] " + ", ".join(f"{k} -> {v}" for k, _p, v in applied))
|
|
|
|
embodiment_type = args.get("embodiment")
|
|
with open(Path(CONFIGS_PATH) / "_embodiment_config.yml", "r", encoding="utf-8") as f:
|
|
embodiment_types = yaml.safe_load(f)
|
|
|
|
def get_embodiment_file(embodiment: str) -> str:
|
|
if embodiment not in embodiment_types:
|
|
sys.exit(
|
|
f"[config] embodiment '{embodiment}' not defined in _embodiment_config.yml; "
|
|
f"available: {sorted(embodiment_types)}"
|
|
)
|
|
robot_file = embodiment_types[embodiment]["file_path"]
|
|
if robot_file is None:
|
|
raise RuntimeError("No embodiment files")
|
|
return robot_file
|
|
|
|
# head 相机分辨率注入 (回灌视频尺寸), 与官方一致
|
|
with open(Path(CONFIGS_PATH) / "_camera_config.yml", "r", encoding="utf-8") as f:
|
|
camera_config = yaml.safe_load(f)
|
|
head_camera_type = args["camera"]["head_camera_type"]
|
|
args["head_camera_h"] = camera_config[head_camera_type]["h"]
|
|
args["head_camera_w"] = camera_config[head_camera_type]["w"]
|
|
|
|
if len(embodiment_type) == 1:
|
|
args["left_robot_file"] = get_embodiment_file(embodiment_type[0])
|
|
args["right_robot_file"] = get_embodiment_file(embodiment_type[0])
|
|
args["dual_arm_embodied"] = True
|
|
embodiment_name = str(embodiment_type[0])
|
|
elif len(embodiment_type) == 3:
|
|
args["left_robot_file"] = get_embodiment_file(embodiment_type[0])
|
|
args["right_robot_file"] = get_embodiment_file(embodiment_type[1])
|
|
args["embodiment_dis"] = embodiment_type[2]
|
|
args["dual_arm_embodied"] = False
|
|
embodiment_name = str(embodiment_type[0]) + "+" + str(embodiment_type[1])
|
|
else:
|
|
raise ValueError("embodiment items should be 1 or 3")
|
|
|
|
args["left_embodiment_config"] = get_embodiment_config(args["left_robot_file"])
|
|
args["right_embodiment_config"] = get_embodiment_config(args["right_robot_file"])
|
|
|
|
# 改动点 3: front_camera_type 注入与采集同源
|
|
config_override.inject_front_camera_type(args, args.get("camera", {}).get("front_camera_type"))
|
|
config_override.validate_camera_types(MERGED_CONFIG_DIR, args)
|
|
|
|
# 改动点 5: SAVE_VIDEO 开关 (覆盖预设 eval_video_log)
|
|
if hp["save_video"]:
|
|
args["eval_video_log"] = True
|
|
else:
|
|
args["eval_video_log"] = False
|
|
|
|
args["embodiment_name"] = embodiment_name
|
|
return args
|
|
|
|
|
|
# ============================================================
|
|
# 指令构造 / 会话通知 (来源: eval_policy_xpolicylab.py)
|
|
# ============================================================
|
|
|
|
def build_instruction(args: dict[str, Any], episode_info: dict[str, Any], instruction_type: str | None, test_num: int) -> str:
|
|
if not instruction_type:
|
|
return args["task_name"]
|
|
try:
|
|
episode_info_list = [episode_info.get("info", {})]
|
|
results = generate_episode_descriptions(args["task_name"], episode_info_list, test_num)
|
|
candidates = results[0].get(instruction_type)
|
|
if candidates:
|
|
return np.random.choice(candidates)
|
|
except Exception:
|
|
print("Failed to generate episode instruction; using task name as instruction.")
|
|
return args["task_name"]
|
|
|
|
|
|
def reset_policy(model_client) -> None:
|
|
model_client.call(func_name="reset")
|
|
|
|
|
|
def prepare_policy_case(model_client, task_name: str, seed: int, instruction: str, action_type: str) -> None:
|
|
if getattr(model_client, "_robotwin_protocol", None) != "ws":
|
|
return
|
|
try:
|
|
model_client.call(
|
|
func_name="prepare_case",
|
|
obs={
|
|
"task_name": task_name,
|
|
"seed": int(seed),
|
|
"instruction": instruction,
|
|
"action_type": action_type,
|
|
},
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def notify_trial_end(model_client, task_name: str, seed: int, success: bool) -> None:
|
|
if getattr(model_client, "_robotwin_protocol", None) != "ws":
|
|
return
|
|
try:
|
|
model_client.call(
|
|
func_name="trial_end",
|
|
obs={
|
|
"task_name": task_name,
|
|
"seed": int(seed),
|
|
"success": bool(success),
|
|
},
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def is_episode_end(task_env) -> bool:
|
|
return task_env.eval_success or task_env.take_action_cnt >= task_env.step_lim
|
|
|
|
|
|
# ============================================================
|
|
# obs 转换 (来源: eval_policy_xpolicylab.py; 改动点 2: camera_map + 删 observer 分支)
|
|
# ============================================================
|
|
|
|
def robotwin_obs_to_xpolicylab(
|
|
observation: Mapping[str, Any],
|
|
*,
|
|
instruction: str | None = None,
|
|
env_idx: int = 0,
|
|
frequency: int = 30,
|
|
task_env: Any | None = None,
|
|
) -> dict[str, Any]:
|
|
instruction_text = instruction or observation.get("language") or ""
|
|
return {
|
|
"data_format_version": "v1.0",
|
|
"instruction": instruction_text,
|
|
"instructions": [instruction_text] if instruction_text else [],
|
|
"env_idx": int(env_idx),
|
|
"vision": convert_vision(observation),
|
|
"state": convert_state(observation, task_env=task_env),
|
|
"additional_info": {"frequency": int(frequency)},
|
|
}
|
|
|
|
|
|
def convert_vision(observation: Mapping[str, Any]) -> dict[str, Any]:
|
|
source = observation.get("observation", {})
|
|
vision = {}
|
|
for robotwin_name, xpolicylab_name in CAMERA_NAME_MAP.items():
|
|
if robotwin_name in source:
|
|
vision[xpolicylab_name] = convert_camera(source[robotwin_name])
|
|
# 改动点 2: observer third_view_rgb 分支已删除 (obs 的 cam_third_view 一律来自 front_camera)
|
|
return vision
|
|
|
|
|
|
def convert_camera(camera: Mapping[str, Any]) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
if "rgb" in camera:
|
|
color = np.asarray(camera["rgb"])
|
|
result["color"] = color
|
|
result["shape"] = tuple(color.shape[:2])
|
|
if "depth" in camera:
|
|
result["depth"] = np.asarray(camera["depth"])
|
|
if "intrinsic_cv" in camera:
|
|
result["intrinsic_matrix"] = np.asarray(camera["intrinsic_cv"], dtype=np.float32)
|
|
elif "intrinsic_matrix" in camera:
|
|
result["intrinsic_matrix"] = np.asarray(camera["intrinsic_matrix"], dtype=np.float32)
|
|
if "extrinsic_cv" in camera:
|
|
result["extrinsics_matrix"] = np.asarray(camera["extrinsic_cv"], dtype=np.float32)
|
|
elif "extrinsics_matrix" in camera:
|
|
result["extrinsics_matrix"] = np.asarray(camera["extrinsics_matrix"], dtype=np.float32)
|
|
if "shape" in camera:
|
|
result["shape"] = tuple(camera["shape"])
|
|
return result
|
|
|
|
|
|
def convert_state(observation: Mapping[str, Any], *, task_env: Any | None = None) -> dict[str, Any]:
|
|
state: dict[str, Any] = {}
|
|
joint_action = observation.get("joint_action", {})
|
|
endpose = observation.get("endpose", {})
|
|
|
|
if "left_arm" in joint_action:
|
|
state["left_arm_joint_state"] = np.asarray(joint_action["left_arm"], dtype=np.float32)
|
|
if "left_gripper" in joint_action:
|
|
state["left_ee_joint_state"] = np.asarray([joint_action["left_gripper"]], dtype=np.float32)
|
|
elif "left_gripper" in endpose:
|
|
state["left_ee_joint_state"] = np.asarray([endpose["left_gripper"]], dtype=np.float32)
|
|
|
|
if "right_arm" in joint_action:
|
|
state["right_arm_joint_state"] = np.asarray(joint_action["right_arm"], dtype=np.float32)
|
|
if "right_gripper" in joint_action:
|
|
state["right_ee_joint_state"] = np.asarray([joint_action["right_gripper"]], dtype=np.float32)
|
|
elif "right_gripper" in endpose:
|
|
state["right_ee_joint_state"] = np.asarray([endpose["right_gripper"]], dtype=np.float32)
|
|
|
|
if "left_endpose" in endpose:
|
|
state["left_ee_pose"] = np.asarray(endpose["left_endpose"], dtype=np.float32)
|
|
if "right_endpose" in endpose:
|
|
state["right_ee_pose"] = np.asarray(endpose["right_endpose"], dtype=np.float32)
|
|
|
|
if task_env is not None and hasattr(task_env, "robot"):
|
|
robot = task_env.robot
|
|
if hasattr(robot, "get_left_tcp_pose"):
|
|
state["left_tcp_pose"] = np.asarray(robot.get_left_tcp_pose(), dtype=np.float32)
|
|
if hasattr(robot, "get_right_tcp_pose"):
|
|
state["right_tcp_pose"] = np.asarray(robot.get_right_tcp_pose(), dtype=np.float32)
|
|
return state
|
|
|
|
|
|
# ============================================================
|
|
# action 归一 (来源: eval_policy_xpolicylab.py)
|
|
# ============================================================
|
|
|
|
def normalize_action_chunk(actions: Any) -> list[Any]:
|
|
if actions is None:
|
|
return []
|
|
if isinstance(actions, Mapping) and "actions" in actions:
|
|
return normalize_action_chunk(actions["actions"])
|
|
if isinstance(actions, Mapping):
|
|
return [actions]
|
|
if isinstance(actions, np.ndarray):
|
|
if actions.ndim == 0:
|
|
raise ValueError("Scalar actions are not supported.")
|
|
if actions.ndim == 1:
|
|
return [actions]
|
|
return [row for row in actions]
|
|
if isinstance(actions, Sequence) and not isinstance(actions, (str, bytes, bytearray)):
|
|
if len(actions) == 0:
|
|
return []
|
|
arr = np.asarray(actions)
|
|
if arr.dtype != object and arr.ndim == 1:
|
|
return [arr]
|
|
return list(actions)
|
|
raise TypeError(f"Unsupported action response type: {type(actions)!r}")
|
|
|
|
|
|
def normalize_robotwin_action_type(action_type: str) -> str:
|
|
normalized = action_type.lower()
|
|
if normalized in {"joint", "qpos"}:
|
|
return "qpos"
|
|
if normalized in {"ee", "endpose"}:
|
|
return "ee"
|
|
raise ValueError(f"Unsupported action_type: {action_type!r}. Use 'joint' or 'ee'.")
|
|
|
|
|
|
def action_array(action, keys, *, fallback, name, expected_dim=None) -> np.ndarray:
|
|
value = first_present(action, keys)
|
|
if value is None:
|
|
if fallback is None:
|
|
raise KeyError(f"Missing {name}; tried keys: {keys}")
|
|
arr = fallback
|
|
else:
|
|
arr = np.asarray(value, dtype=np.float32).reshape(-1)
|
|
if expected_dim is not None and arr.shape[0] != expected_dim:
|
|
raise ValueError(f"{name} must have dim {expected_dim}, got {arr.shape[0]}")
|
|
return arr
|
|
|
|
|
|
def action_gripper(action, keys, *, fallback, name) -> float:
|
|
value = first_present(action, keys)
|
|
if value is None:
|
|
if fallback is None:
|
|
raise KeyError(f"Missing {name}; tried keys: {keys}")
|
|
return float(fallback)
|
|
arr = np.asarray(value, dtype=np.float32).reshape(-1)
|
|
if arr.shape[0] != 1:
|
|
raise ValueError(f"{name} must be scalar for RoboTwin, got shape {arr.shape}")
|
|
return float(arr[0])
|
|
|
|
|
|
def first_present(data, keys):
|
|
for key in keys:
|
|
if key in data:
|
|
return data[key]
|
|
return None
|
|
|
|
|
|
def current_joint_arm(observation, side: str):
|
|
if observation is None:
|
|
return None
|
|
joint_action = observation.get("joint_action", {})
|
|
key = f"{side}_arm"
|
|
if key not in joint_action:
|
|
return None
|
|
return np.asarray(joint_action[key], dtype=np.float32).reshape(-1)
|
|
|
|
|
|
def current_gripper(observation, side: str):
|
|
if observation is None:
|
|
return None
|
|
joint_action = observation.get("joint_action", {})
|
|
key = f"{side}_gripper"
|
|
if key in joint_action:
|
|
return float(np.asarray(joint_action[key]).reshape(-1)[0])
|
|
endpose = observation.get("endpose", {})
|
|
if key in endpose:
|
|
return float(np.asarray(endpose[key]).reshape(-1)[0])
|
|
return None
|
|
|
|
|
|
def current_ee_pose(observation, side: str):
|
|
if observation is None:
|
|
return None
|
|
endpose = observation.get("endpose", {})
|
|
key = f"{side}_endpose"
|
|
if key not in endpose:
|
|
return None
|
|
return np.asarray(endpose[key], dtype=np.float32).reshape(-1)
|
|
|
|
|
|
def xpolicylab_action_to_robotwin(
|
|
action: Any,
|
|
*,
|
|
action_type: str,
|
|
current_observation=None,
|
|
):
|
|
robotwin_action_type = normalize_robotwin_action_type(action_type)
|
|
if not isinstance(action, Mapping):
|
|
return np.asarray(action, dtype=np.float32).reshape(-1), robotwin_action_type
|
|
|
|
resolved_action_type = action.get("action_type", action_type)
|
|
robotwin_action_type = normalize_robotwin_action_type(str(resolved_action_type))
|
|
|
|
if robotwin_action_type == "qpos":
|
|
left_arm = action_array(
|
|
action,
|
|
("left_arm_joint_state", "left_arm_joint", "left_joint_state", "arm_joint_state", "joint_state"),
|
|
fallback=current_joint_arm(current_observation, "left"),
|
|
name="left arm joint action",
|
|
)
|
|
right_arm = action_array(
|
|
action,
|
|
("right_arm_joint_state", "right_arm_joint", "right_joint_state"),
|
|
fallback=current_joint_arm(current_observation, "right"),
|
|
name="right arm joint action",
|
|
)
|
|
left_gripper = action_gripper(
|
|
action,
|
|
("left_ee_joint_state", "left_gripper", "left_gripper_pos", "ee_joint_state"),
|
|
fallback=current_gripper(current_observation, "left"),
|
|
name="left gripper action",
|
|
)
|
|
right_gripper = action_gripper(
|
|
action,
|
|
("right_ee_joint_state", "right_gripper", "right_gripper_pos"),
|
|
fallback=current_gripper(current_observation, "right"),
|
|
name="right gripper action",
|
|
)
|
|
else:
|
|
left_arm = action_array(
|
|
action,
|
|
("left_ee_pose", "left_endpose", "ee_pose"),
|
|
fallback=current_ee_pose(current_observation, "left"),
|
|
name="left ee action",
|
|
expected_dim=7,
|
|
)
|
|
right_arm = action_array(
|
|
action,
|
|
("right_ee_pose", "right_endpose"),
|
|
fallback=current_ee_pose(current_observation, "right"),
|
|
name="right ee action",
|
|
expected_dim=7,
|
|
)
|
|
left_gripper = action_gripper(
|
|
action,
|
|
("left_ee_joint_state", "left_gripper", "left_gripper_pos", "ee_joint_state"),
|
|
fallback=current_gripper(current_observation, "left"),
|
|
name="left gripper action",
|
|
)
|
|
right_gripper = action_gripper(
|
|
action,
|
|
("right_ee_joint_state", "right_gripper", "right_gripper_pos"),
|
|
fallback=current_gripper(current_observation, "right"),
|
|
name="right gripper action",
|
|
)
|
|
|
|
flat_action = np.concatenate(
|
|
[
|
|
left_arm.astype(np.float32),
|
|
np.asarray([left_gripper], dtype=np.float32),
|
|
right_arm.astype(np.float32),
|
|
np.asarray([right_gripper], dtype=np.float32),
|
|
]
|
|
)
|
|
return flat_action, robotwin_action_type
|
|
|
|
|
|
# ============================================================
|
|
# 单 episode 评测 (来源: run_one_batch_episode, 单进程化)
|
|
# ============================================================
|
|
|
|
def _load_official_step_limit(task_name: str) -> int:
|
|
"""官方 _eval_step_limit.yml 的每任务步数上限 (缺省 400)。"""
|
|
import yaml
|
|
|
|
path = Path(CONFIGS_PATH) / "_eval_step_limit.yml"
|
|
try:
|
|
with open(path, encoding="utf-8") as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return int(data.get(task_name, 400))
|
|
except (OSError, TypeError, ValueError):
|
|
return 400
|
|
|
|
|
|
def run_episode(
|
|
episode_id: int,
|
|
seed_value: int,
|
|
task_name: str,
|
|
task_env,
|
|
args: dict[str, Any],
|
|
model_client,
|
|
hp: dict,
|
|
instruction_type: str | None,
|
|
video_size: str | None,
|
|
) -> dict[str, Any]:
|
|
action_type = hp["action_type"]
|
|
frequency = hp["frequency"]
|
|
expert_check = hp["expert_check"]
|
|
render_freq = args["render_freq"]
|
|
args["render_freq"] = 0
|
|
episode_info = {"info": {}}
|
|
|
|
if expert_check:
|
|
try:
|
|
task_env.setup_demo(now_ep_num=episode_id, seed=seed_value, is_test=True, **args)
|
|
episode_info = task_env.play_once()
|
|
task_env.close_env()
|
|
except UnStableError:
|
|
task_env.close_env()
|
|
args["render_freq"] = render_freq
|
|
return {"type": "seed_skipped", "seed": seed_value, "reason": "unstable"}
|
|
except Exception as e:
|
|
task_env.close_env()
|
|
args["render_freq"] = render_freq
|
|
print(f"error occurs during expert check! seed={seed_value} err={type(e).__name__}: {e}")
|
|
return {"type": "seed_skipped", "seed": seed_value, "reason": "expert_error"}
|
|
|
|
if expert_check and not (task_env.plan_success and task_env.check_success()):
|
|
args["render_freq"] = render_freq
|
|
return {"type": "seed_skipped", "seed": seed_value, "reason": "expert_failed"}
|
|
|
|
args["render_freq"] = render_freq
|
|
try:
|
|
task_env.setup_demo(now_ep_num=episode_id, seed=seed_value, is_test=True, **args)
|
|
except UnStableError:
|
|
try:
|
|
task_env.close_env()
|
|
except Exception:
|
|
pass
|
|
print(f"skip unstable seed={seed_value} (eval setup)")
|
|
return {"type": "seed_skipped", "seed": seed_value, "reason": "unstable"}
|
|
|
|
# 改动点 4: STEP_LIMIT 绝对值覆盖 (is_episode_end 与 take_action 双消费点同时生效);
|
|
# 空值时回退官方 _eval_step_limit.yml 每任务值 (任务 env 默认 step_lim=None,
|
|
# 缺失时 is_episode_end 的 >= 比较直接 TypeError, 改动点 4 声明的回退此前未实现)
|
|
if hp["step_limit"] > 0:
|
|
task_env.step_lim = int(hp["step_limit"])
|
|
elif getattr(task_env, "step_lim", None) is None:
|
|
task_env.step_lim = _load_official_step_limit(task_name)
|
|
|
|
instruction = build_instruction(args, episode_info, instruction_type, hp["test_num"])
|
|
task_env.set_instruction(instruction=instruction)
|
|
|
|
if getattr(task_env, "eval_video_path", None) is not None:
|
|
ffmpeg = subprocess.Popen(
|
|
[
|
|
"ffmpeg",
|
|
"-y",
|
|
"-loglevel", "error",
|
|
"-f", "rawvideo",
|
|
"-pixel_format", "rgb24",
|
|
"-video_size", video_size,
|
|
"-framerate", "10",
|
|
"-i", "-",
|
|
"-pix_fmt", "yuv420p",
|
|
"-vcodec", "libx264",
|
|
"-crf", "23",
|
|
f"{task_env.eval_video_path}/episode{episode_id}.mp4",
|
|
],
|
|
stdin=subprocess.PIPE,
|
|
)
|
|
task_env._set_eval_video_ffmpeg(ffmpeg)
|
|
|
|
succ = False
|
|
prepare_policy_case(model_client, task_name, seed_value, instruction, action_type)
|
|
reset_policy(model_client)
|
|
try:
|
|
while not is_episode_end(task_env):
|
|
observation = task_env.get_obs()
|
|
xpl_obs = robotwin_obs_to_xpolicylab(
|
|
observation,
|
|
instruction=task_env.get_instruction(),
|
|
env_idx=0,
|
|
frequency=frequency,
|
|
task_env=task_env,
|
|
)
|
|
model_client.call(func_name="update_obs", obs=xpl_obs)
|
|
action_chunk = normalize_action_chunk(model_client.call(func_name="get_action"))
|
|
if len(action_chunk) == 0:
|
|
raise RuntimeError("Policy returned an empty action chunk.")
|
|
|
|
# ACTION_STRIDE: 每 chunk 只执行前 N 个动作后即重新请求动作块 (0 = 整块执行)
|
|
n_exec = len(action_chunk)
|
|
if hp["action_stride"] > 0:
|
|
n_exec = min(hp["action_stride"], len(action_chunk))
|
|
|
|
for action_idx, action in enumerate(action_chunk):
|
|
flat_action, robotwin_action_type = xpolicylab_action_to_robotwin(
|
|
action,
|
|
action_type=action_type,
|
|
current_observation=observation,
|
|
)
|
|
task_env.take_action(flat_action, action_type=robotwin_action_type)
|
|
|
|
if task_env.eval_success:
|
|
succ = True
|
|
break
|
|
if is_episode_end(task_env) or action_idx + 1 >= n_exec:
|
|
break
|
|
|
|
# 块内不逐动作发送 update_obs: 外层循环每次 get_action 前都会重新取 obs
|
|
# 并 update, 块内逐动作发送属纯冗余; 一并省去块内逐动作的相机渲染与
|
|
# 网络带宽 (远程策略服务场景带宽降幅显著)。块内动作转换 fallback 使用
|
|
# 块首 obs, 数组动作不受影响。
|
|
|
|
if succ:
|
|
break
|
|
except Exception:
|
|
print(f"\n\033[91mPolicy rollout error (task {task_name}, seed {seed_value}):\033[0m")
|
|
print(traceback.format_exc())
|
|
|
|
if getattr(task_env, "eval_video_path", None) is not None:
|
|
task_env._del_eval_video_ffmpeg()
|
|
|
|
notify_trial_end(model_client, task_name, seed_value, succ)
|
|
|
|
task_env.close_env()
|
|
if task_env.render_freq:
|
|
task_env.viewer.close()
|
|
|
|
return {"type": "episode_done", "episode_id": episode_id, "seed": seed_value, "success": bool(succ)}
|
|
|
|
|
|
def eval_one_task(task_name: str, hp: dict, model_client) -> tuple[int, int]:
|
|
args = load_task_args(task_name, hp)
|
|
instruction_type = resolve_eval_instruction(hp, args)
|
|
args["eval_instruction"] = instruction_type
|
|
|
|
output_dir = Path(hp["output_dir"])
|
|
task_result_dir = output_dir / "eval_result" / task_name
|
|
task_result_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 改动点 5: 回灌视频目录 $OUTPUT_DIR/videos/<task>/
|
|
video_size = None
|
|
if args["eval_video_log"]:
|
|
camera_config = get_camera_config(args["camera"]["head_camera_type"])
|
|
video_size = str(camera_config["w"]) + "x" + str(camera_config["h"])
|
|
args["eval_video_save_dir"] = str(output_dir / "videos" / task_name)
|
|
(output_dir / "videos" / task_name).mkdir(parents=True, exist_ok=True)
|
|
|
|
print(f"\n\033[96m===== Eval Task: {task_name} (instruction={instruction_type}) =====\033[0m")
|
|
|
|
task_env = class_decorator(task_name)
|
|
seed = int(hp["seed"])
|
|
st_seed = 100000 * (1 + seed)
|
|
test_num = int(hp["test_num"])
|
|
max_seed_attempts = int(hp["max_seed_attempts"]) or (test_num * 50)
|
|
|
|
episode_id, success_num, completed, skipped = 0, 0, 0, 0
|
|
for seed_value in range(st_seed, st_seed + max_seed_attempts):
|
|
if completed >= test_num:
|
|
break
|
|
result = run_episode(
|
|
episode_id, seed_value, task_name, task_env, args, model_client, hp,
|
|
instruction_type, video_size,
|
|
)
|
|
if result["type"] == "seed_skipped":
|
|
skipped += 1
|
|
continue
|
|
episode_id += 1
|
|
completed += 1
|
|
success_num += int(bool(result["success"]))
|
|
print(
|
|
f"\033[93m{task_name}\033[0m | episode={result['episode_id']} | seed={result['seed']} | "
|
|
f"success={result['success']} | task success: \033[96m{success_num}/{completed}\033[0m"
|
|
)
|
|
report_utils.emit_progress(
|
|
f"eval:{task_name}",
|
|
episode=completed,
|
|
maxEpisode=test_num,
|
|
success=success_num,
|
|
)
|
|
|
|
try:
|
|
task_env.close_env()
|
|
except Exception:
|
|
pass
|
|
|
|
if completed == 0:
|
|
raise RuntimeError(
|
|
f"Eval completed zero episodes for task {task_name}. skipped_seeds={skipped}, "
|
|
f"max_seed_attempts={max_seed_attempts}"
|
|
)
|
|
|
|
with open(task_result_dir / "_result.txt", "w", encoding="utf-8") as file:
|
|
file.write(f"Instruction Type: {instruction_type}\n")
|
|
file.write(f"Skipped Seeds: {skipped}\n")
|
|
file.write(f"{success_num}/{completed}\n")
|
|
|
|
return success_num, completed
|
|
|
|
|
|
# ============================================================
|
|
# Main: 多任务循环 + 汇总 (改动点 1/7)
|
|
# ============================================================
|
|
|
|
def main() -> None:
|
|
hp = read_hyperparams()
|
|
np.random.seed(int(hp["seed"])) # 指令随机选择可复现
|
|
|
|
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S")
|
|
tasks_summary: dict[str, str] = {}
|
|
total_success, total_completed = 0, 0
|
|
|
|
model_client = build_policy_client(hp)
|
|
try:
|
|
for task_name in hp["task_names"]:
|
|
suc, done = eval_one_task(task_name, hp, model_client)
|
|
tasks_summary[task_name] = f"{suc}/{done}"
|
|
total_success += suc
|
|
total_completed += done
|
|
finally:
|
|
close_policy_client(model_client)
|
|
|
|
overall = round(total_success / total_completed, 4) if total_completed else 0.0
|
|
summary = {
|
|
"tasks": tasks_summary,
|
|
"overall": overall,
|
|
"config": {
|
|
"timestamp": current_time,
|
|
"policy_server": f"{hp['policy_server_ip']}:{hp['policy_server_port']}",
|
|
"protocol": hp["protocol"],
|
|
"policy_name": hp["policy_name"],
|
|
"ckpt_setting": hp["ckpt_setting"],
|
|
"task_config": hp["task_config"],
|
|
"test_num": hp["test_num"],
|
|
"seed": hp["seed"],
|
|
"action_type": hp["action_type"],
|
|
"frequency": hp["frequency"],
|
|
},
|
|
}
|
|
output_dir = Path(hp["output_dir"])
|
|
(output_dir / "eval_summary.json").write_text(
|
|
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
print(f"Overall success rate: {total_success}/{total_completed} = {round(overall * 100, 1)}%")
|
|
|
|
report_utils.write_artifacts(
|
|
output_dir,
|
|
[
|
|
{"name": "eval_result", "type": "file", "portId": "eval_result", "path": str(output_dir / "eval_result")},
|
|
{"name": "eval_summary", "type": "json", "portId": "eval_report", "path": str(output_dir / "eval_summary.json")},
|
|
],
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sapien_render_check()
|
|
main()
|