887 lines
35 KiB
Python

# 来源: RoboTwin仓库 scripts/eval_policy_xpolicylab.py @ 30954692d06ba7e89f7a6b76064f4062c488fa81
# 改动点:
# 1) 多任务循环由本文件承担 (TASK_NAME 逗号分隔逐任务顺序执行); 评测固定单进程
# (官方批模式开关 EVAL_BATCH 语义移除), 去多进程批调度与共享内存依赖
# 2) 相机映射改用 ws_common.camera_map (补 front_camera -> cam_third_view; 删 observer
# third_view_rgb 分支: 评测 obs 的 cam_third_view 一律来自 front_camera, 与采集对齐)
# 3) front_camera_type 注入与采集侧同源 (ws_common.config_override, 预设扩展键或超参)
# 4) STEP_LIMIT 绝对值覆盖 (空值用官方 _eval_step_limit.yml 每任务值)
# 5) SAVE_VIDEO 开关 (回灌视频写 $OUTPUT_DIR/videos/<task>/episodeN.mp4)
# 6) 启动渲染自检, 失败非 0 退出 (官方 Sapien_TEST 失败退出码为 0, 此处修正)
# 7) 逐任务结果写 $OUTPUT_DIR/eval_result/<task>/_result.txt, 汇总 eval_summary.json 写
# $OUTPUT_DIR (禁用 result.json 文件名), 结束写 .artifacts.json; 超参全部环境变量注入
# 另: legacy_tcp 分支 import 路径修正为 client_server.tcp.model_client (vendored 树实际位置)
from __future__ import annotations
import importlib
import json
import os
import subprocess
import sys
import traceback
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
import yaml
APP_SRC = Path(os.environ.get("APP_SRC", "/app/src"))
ROBOTWIN_ROOT = Path(os.environ.get("ROBOTWIN_ROOT", str(APP_SRC / "RoboTwin")))
PRESETS_DIR = Path(os.environ.get("PRESETS_DIR", str(APP_SRC / "presets")))
DEFAULT_XPOLICYLAB_ROOT = APP_SRC / "vendor" / "XPolicyLab"
for _p in (str(APP_SRC), str(ROBOTWIN_ROOT), str(ROBOTWIN_ROOT / "description" / "utils")):
if _p not in sys.path:
sys.path.insert(0, _p)
os.chdir(ROBOTWIN_ROOT)
from ws_common import config_override, report_utils # noqa: E402
from ws_common.camera_map import CAMERA_NAME_MAP # noqa: E402
# 配置目录合并 + CONFIGS_PATH 改写 (须先于 envs 任务类加载)
MERGED_CONFIG_DIR = config_override.build_merged_config_dir(ROBOTWIN_ROOT, PRESETS_DIR)
CONFIGS_PATH = config_override.install_config_dir(MERGED_CONFIG_DIR)
from envs.utils.create_actor import UnStableError # noqa: E402
from generate_episode_instructions import generate_episode_descriptions # noqa: E402
# ============================================================
# 渲染自检 (改动点 6: 失败非 0)
# ============================================================
def sapien_render_check() -> None:
import sapien.core as sapien # noqa: PLC0415
from sapien.render import set_global_config # noqa: PLC0415
try:
engine = sapien.Engine()
set_global_config(max_num_materials=50000, max_num_textures=50000)
renderer = sapien.SapienRenderer()
engine.set_renderer(renderer)
sapien.render.set_camera_shader_dir("rt")
sapien.render.set_ray_tracing_samples_per_pixel(32)
sapien.render.set_ray_tracing_path_depth(8)
sapien.render.set_ray_tracing_denoiser("oidn")
scene = engine.create_scene(sapien.SceneConfig())
del scene, renderer, engine
except SystemExit:
raise
except Exception as exc:
print(f"[render-check] FAILED: {type(exc).__name__}: {exc}", file=sys.stderr)
sys.exit(1)
print("[render-check] Render Well")
# ============================================================
# 超参
# ============================================================
def read_hyperparams() -> dict:
env = os.environ
hp: dict[str, Any] = {
"policy_server_ip": env.get("POLICY_SERVER_IP", ""),
"policy_server_port": env.get("POLICY_SERVER_PORT", ""),
"protocol": env.get("PROTOCOL", "") or "ws",
"policy_name": env.get("POLICY_NAME", "") or "custom_policy",
"ckpt_setting": env.get("CKPT_SETTING", "") or "default",
"task_names": [t.strip() for t in env.get("TASK_NAME", "").split(",") if t.strip()],
"task_config": env.get("TASK_CONFIG", "") or "demo_clean",
"instruction_type": env.get("INSTRUCTION_TYPE", "") or None,
"action_type": env.get("ACTION_TYPE", "") or "joint",
"step_limit": config_override.parse_typed(env.get("STEP_LIMIT", "") or "0", "int", "STEP_LIMIT"),
"action_stride": config_override.parse_typed(env.get("ACTION_STRIDE", "") or "0", "int", "ACTION_STRIDE"),
"test_num": config_override.parse_typed(env.get("TEST_NUM", "") or "5", "int", "TEST_NUM"),
"seed": config_override.parse_typed(env.get("SEED", "") or "0", "int", "SEED"),
"expert_check": config_override.parse_typed(env.get("EXPERT_CHECK", "") or "true", "bool", "EXPERT_CHECK"),
"frequency": config_override.parse_typed(env.get("FREQUENCY", "") or "30", "int", "FREQUENCY"),
"max_seed_attempts": config_override.parse_typed(env.get("MAX_SEED_ATTEMPTS", "") or "0", "int", "MAX_SEED_ATTEMPTS"),
"save_video": config_override.parse_typed(env.get("SAVE_VIDEO", "") or "true", "bool", "SAVE_VIDEO"),
"output_dir": env.get("OUTPUT_DIR", "") or os.getcwd(),
}
if not hp["policy_server_ip"] or not hp["policy_server_port"]:
sys.exit("[hyperparam] POLICY_SERVER_IP and POLICY_SERVER_PORT are required")
if not hp["task_names"]:
sys.exit("[hyperparam] TASK_NAME is required (comma-separated task list)")
if hp["protocol"] not in ("ws", "legacy_tcp"):
sys.exit(f"[hyperparam] PROTOCOL must be ws or legacy_tcp, got {hp['protocol']!r}")
if hp["action_type"] not in ("joint", "ee", "qpos", "endpose"):
sys.exit(f"[hyperparam] ACTION_TYPE must be joint or ee, got {hp['action_type']!r}")
return hp
# ============================================================
# 策略客户端 (来源: eval_policy_xpolicylab.py build_policy_client, tcp import 路径修正)
# ============================================================
def add_xpolicylab_paths(xpolicylab_root) -> None:
if not xpolicylab_root:
return
root = os.path.abspath(os.fspath(xpolicylab_root))
for path in (root, os.path.join(root, "integrations")):
if path not in sys.path:
sys.path.insert(0, path)
def build_policy_client(hp: dict):
protocol = hp["protocol"]
add_xpolicylab_paths(os.environ.get("XPOLICYLAB_ROOT", str(DEFAULT_XPOLICYLAB_ROOT)))
host = hp["policy_server_ip"]
port = int(hp["policy_server_port"])
current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S_%f")
if protocol == "ws":
from client_server.ws import WsModelClient # noqa: PLC0415
client = WsModelClient(
url=f"ws://{host}:{port}",
evaluation_id=f"robotwin-eval-{current_time}",
trial_id="robotwin-trial",
action_case_id="robotwin-case",
repeat_index=None,
)
client._robotwin_protocol = "ws"
return client
from client_server.tcp.model_client import ModelClient # noqa: PLC0415
client = ModelClient(host=host, port=port)
client._robotwin_protocol = protocol
return client
def close_policy_client(model_client) -> None:
close = getattr(model_client, "close", None)
if callable(close):
close()
# ============================================================
# 任务配置加载 (与采集侧同源: 合并目录 + 覆盖 + front 注入)
# ============================================================
def class_decorator(task_name: str):
envs_module = importlib.import_module(f"envs.{task_name}")
try:
env_class = getattr(envs_module, task_name)
return env_class()
except Exception as exc:
raise SystemExit(f"No Task: {task_name} ({exc})") from exc
def get_camera_config(camera_type: str) -> dict[str, Any]:
camera_config_path = Path(CONFIGS_PATH) / "_camera_config.yml"
with open(camera_config_path, "r", encoding="utf-8") as f:
args = yaml.safe_load(f)
if camera_type not in args:
raise KeyError(f"camera {camera_type} is not defined")
return args[camera_type]
def get_embodiment_config(robot_file: str) -> dict[str, Any]:
robot_config_file = os.path.join(robot_file, "config.yml")
with open(robot_config_file, "r", encoding="utf-8") as f:
return yaml.safe_load(f)
def ensure_xpolicylab_observation_flags(args: dict[str, Any]) -> None:
data_type = args.setdefault("data_type", {})
data_type["rgb"] = True
data_type["qpos"] = True
data_type["endpose"] = True
def resolve_eval_instruction(hp: dict[str, Any], task_args: dict[str, Any]) -> str:
instruction_type = hp.get("instruction_type")
if not instruction_type:
if "eval_instruction" not in task_args:
raise ValueError("task config must define eval_instruction as 'seen' or 'unseen'")
instruction_type = task_args["eval_instruction"]
instruction_type = str(instruction_type).strip().lower()
if instruction_type not in {"seen", "unseen"}:
raise ValueError(f"eval instruction must be 'seen' or 'unseen', got {instruction_type!r}")
return instruction_type
def load_task_args(task_name: str, hp: dict) -> dict[str, Any]:
config_path = Path(CONFIGS_PATH) / f"{hp['task_config']}.yml"
if not config_path.is_file():
available = sorted(p.stem for p in Path(CONFIGS_PATH).glob("*.yml") if not p.name.startswith("_"))
sys.exit(f"[config] task config '{hp['task_config']}' not found; available presets: {available}")
with open(config_path, "r", encoding="utf-8") as f:
args = yaml.safe_load(f)
args["task_name"] = task_name
args["task_config"] = hp["task_config"]
ensure_xpolicylab_observation_flags(args)
# 超参覆盖 (env > 预设 yml > 内置默认), 与采集侧完全相同的一组覆盖键
applied = config_override.apply_env_overrides(args)
if applied:
print("[override] " + ", ".join(f"{k} -> {v}" for k, _p, v in applied))
embodiment_type = args.get("embodiment")
with open(Path(CONFIGS_PATH) / "_embodiment_config.yml", "r", encoding="utf-8") as f:
embodiment_types = yaml.safe_load(f)
def get_embodiment_file(embodiment: str) -> str:
if embodiment not in embodiment_types:
sys.exit(
f"[config] embodiment '{embodiment}' not defined in _embodiment_config.yml; "
f"available: {sorted(embodiment_types)}"
)
robot_file = embodiment_types[embodiment]["file_path"]
if robot_file is None:
raise RuntimeError("No embodiment files")
return robot_file
# head 相机分辨率注入 (回灌视频尺寸), 与官方一致
with open(Path(CONFIGS_PATH) / "_camera_config.yml", "r", encoding="utf-8") as f:
camera_config = yaml.safe_load(f)
head_camera_type = args["camera"]["head_camera_type"]
args["head_camera_h"] = camera_config[head_camera_type]["h"]
args["head_camera_w"] = camera_config[head_camera_type]["w"]
if len(embodiment_type) == 1:
args["left_robot_file"] = get_embodiment_file(embodiment_type[0])
args["right_robot_file"] = get_embodiment_file(embodiment_type[0])
args["dual_arm_embodied"] = True
embodiment_name = str(embodiment_type[0])
elif len(embodiment_type) == 3:
args["left_robot_file"] = get_embodiment_file(embodiment_type[0])
args["right_robot_file"] = get_embodiment_file(embodiment_type[1])
args["embodiment_dis"] = embodiment_type[2]
args["dual_arm_embodied"] = False
embodiment_name = str(embodiment_type[0]) + "+" + str(embodiment_type[1])
else:
raise ValueError("embodiment items should be 1 or 3")
args["left_embodiment_config"] = get_embodiment_config(args["left_robot_file"])
args["right_embodiment_config"] = get_embodiment_config(args["right_robot_file"])
# 改动点 3: front_camera_type 注入与采集同源
config_override.inject_front_camera_type(args, args.get("camera", {}).get("front_camera_type"))
config_override.validate_camera_types(MERGED_CONFIG_DIR, args)
# 改动点 5: SAVE_VIDEO 开关 (覆盖预设 eval_video_log)
if hp["save_video"]:
args["eval_video_log"] = True
else:
args["eval_video_log"] = False
args["embodiment_name"] = embodiment_name
return args
# ============================================================
# 指令构造 / 会话通知 (来源: eval_policy_xpolicylab.py)
# ============================================================
def build_instruction(args: dict[str, Any], episode_info: dict[str, Any], instruction_type: str | None, test_num: int) -> str:
if not instruction_type:
return args["task_name"]
try:
episode_info_list = [episode_info.get("info", {})]
results = generate_episode_descriptions(args["task_name"], episode_info_list, test_num)
candidates = results[0].get(instruction_type)
if candidates:
return np.random.choice(candidates)
except Exception:
print("Failed to generate episode instruction; using task name as instruction.")
return args["task_name"]
def reset_policy(model_client) -> None:
model_client.call(func_name="reset")
def prepare_policy_case(model_client, task_name: str, seed: int, instruction: str, action_type: str) -> None:
if getattr(model_client, "_robotwin_protocol", None) != "ws":
return
try:
model_client.call(
func_name="prepare_case",
obs={
"task_name": task_name,
"seed": int(seed),
"instruction": instruction,
"action_type": action_type,
},
)
except Exception:
pass
def notify_trial_end(model_client, task_name: str, seed: int, success: bool) -> None:
if getattr(model_client, "_robotwin_protocol", None) != "ws":
return
try:
model_client.call(
func_name="trial_end",
obs={
"task_name": task_name,
"seed": int(seed),
"success": bool(success),
},
)
except Exception:
pass
def is_episode_end(task_env) -> bool:
return task_env.eval_success or task_env.take_action_cnt >= task_env.step_lim
# ============================================================
# obs 转换 (来源: eval_policy_xpolicylab.py; 改动点 2: camera_map + 删 observer 分支)
# ============================================================
def robotwin_obs_to_xpolicylab(
observation: Mapping[str, Any],
*,
instruction: str | None = None,
env_idx: int = 0,
frequency: int = 30,
task_env: Any | None = None,
) -> dict[str, Any]:
instruction_text = instruction or observation.get("language") or ""
return {
"data_format_version": "v1.0",
"instruction": instruction_text,
"instructions": [instruction_text] if instruction_text else [],
"env_idx": int(env_idx),
"vision": convert_vision(observation),
"state": convert_state(observation, task_env=task_env),
"additional_info": {"frequency": int(frequency)},
}
def convert_vision(observation: Mapping[str, Any]) -> dict[str, Any]:
source = observation.get("observation", {})
vision = {}
for robotwin_name, xpolicylab_name in CAMERA_NAME_MAP.items():
if robotwin_name in source:
vision[xpolicylab_name] = convert_camera(source[robotwin_name])
# 改动点 2: observer third_view_rgb 分支已删除 (obs 的 cam_third_view 一律来自 front_camera)
return vision
def convert_camera(camera: Mapping[str, Any]) -> dict[str, Any]:
result: dict[str, Any] = {}
if "rgb" in camera:
color = np.asarray(camera["rgb"])
result["color"] = color
result["shape"] = tuple(color.shape[:2])
if "depth" in camera:
result["depth"] = np.asarray(camera["depth"])
if "intrinsic_cv" in camera:
result["intrinsic_matrix"] = np.asarray(camera["intrinsic_cv"], dtype=np.float32)
elif "intrinsic_matrix" in camera:
result["intrinsic_matrix"] = np.asarray(camera["intrinsic_matrix"], dtype=np.float32)
if "extrinsic_cv" in camera:
result["extrinsics_matrix"] = np.asarray(camera["extrinsic_cv"], dtype=np.float32)
elif "extrinsics_matrix" in camera:
result["extrinsics_matrix"] = np.asarray(camera["extrinsics_matrix"], dtype=np.float32)
if "shape" in camera:
result["shape"] = tuple(camera["shape"])
return result
def convert_state(observation: Mapping[str, Any], *, task_env: Any | None = None) -> dict[str, Any]:
state: dict[str, Any] = {}
joint_action = observation.get("joint_action", {})
endpose = observation.get("endpose", {})
if "left_arm" in joint_action:
state["left_arm_joint_state"] = np.asarray(joint_action["left_arm"], dtype=np.float32)
if "left_gripper" in joint_action:
state["left_ee_joint_state"] = np.asarray([joint_action["left_gripper"]], dtype=np.float32)
elif "left_gripper" in endpose:
state["left_ee_joint_state"] = np.asarray([endpose["left_gripper"]], dtype=np.float32)
if "right_arm" in joint_action:
state["right_arm_joint_state"] = np.asarray(joint_action["right_arm"], dtype=np.float32)
if "right_gripper" in joint_action:
state["right_ee_joint_state"] = np.asarray([joint_action["right_gripper"]], dtype=np.float32)
elif "right_gripper" in endpose:
state["right_ee_joint_state"] = np.asarray([endpose["right_gripper"]], dtype=np.float32)
if "left_endpose" in endpose:
state["left_ee_pose"] = np.asarray(endpose["left_endpose"], dtype=np.float32)
if "right_endpose" in endpose:
state["right_ee_pose"] = np.asarray(endpose["right_endpose"], dtype=np.float32)
if task_env is not None and hasattr(task_env, "robot"):
robot = task_env.robot
if hasattr(robot, "get_left_tcp_pose"):
state["left_tcp_pose"] = np.asarray(robot.get_left_tcp_pose(), dtype=np.float32)
if hasattr(robot, "get_right_tcp_pose"):
state["right_tcp_pose"] = np.asarray(robot.get_right_tcp_pose(), dtype=np.float32)
return state
# ============================================================
# action 归一 (来源: eval_policy_xpolicylab.py)
# ============================================================
def normalize_action_chunk(actions: Any) -> list[Any]:
if actions is None:
return []
if isinstance(actions, Mapping) and "actions" in actions:
return normalize_action_chunk(actions["actions"])
if isinstance(actions, Mapping):
return [actions]
if isinstance(actions, np.ndarray):
if actions.ndim == 0:
raise ValueError("Scalar actions are not supported.")
if actions.ndim == 1:
return [actions]
return [row for row in actions]
if isinstance(actions, Sequence) and not isinstance(actions, (str, bytes, bytearray)):
if len(actions) == 0:
return []
arr = np.asarray(actions)
if arr.dtype != object and arr.ndim == 1:
return [arr]
return list(actions)
raise TypeError(f"Unsupported action response type: {type(actions)!r}")
def normalize_robotwin_action_type(action_type: str) -> str:
normalized = action_type.lower()
if normalized in {"joint", "qpos"}:
return "qpos"
if normalized in {"ee", "endpose"}:
return "ee"
raise ValueError(f"Unsupported action_type: {action_type!r}. Use 'joint' or 'ee'.")
def action_array(action, keys, *, fallback, name, expected_dim=None) -> np.ndarray:
value = first_present(action, keys)
if value is None:
if fallback is None:
raise KeyError(f"Missing {name}; tried keys: {keys}")
arr = fallback
else:
arr = np.asarray(value, dtype=np.float32).reshape(-1)
if expected_dim is not None and arr.shape[0] != expected_dim:
raise ValueError(f"{name} must have dim {expected_dim}, got {arr.shape[0]}")
return arr
def action_gripper(action, keys, *, fallback, name) -> float:
value = first_present(action, keys)
if value is None:
if fallback is None:
raise KeyError(f"Missing {name}; tried keys: {keys}")
return float(fallback)
arr = np.asarray(value, dtype=np.float32).reshape(-1)
if arr.shape[0] != 1:
raise ValueError(f"{name} must be scalar for RoboTwin, got shape {arr.shape}")
return float(arr[0])
def first_present(data, keys):
for key in keys:
if key in data:
return data[key]
return None
def current_joint_arm(observation, side: str):
if observation is None:
return None
joint_action = observation.get("joint_action", {})
key = f"{side}_arm"
if key not in joint_action:
return None
return np.asarray(joint_action[key], dtype=np.float32).reshape(-1)
def current_gripper(observation, side: str):
if observation is None:
return None
joint_action = observation.get("joint_action", {})
key = f"{side}_gripper"
if key in joint_action:
return float(np.asarray(joint_action[key]).reshape(-1)[0])
endpose = observation.get("endpose", {})
if key in endpose:
return float(np.asarray(endpose[key]).reshape(-1)[0])
return None
def current_ee_pose(observation, side: str):
if observation is None:
return None
endpose = observation.get("endpose", {})
key = f"{side}_endpose"
if key not in endpose:
return None
return np.asarray(endpose[key], dtype=np.float32).reshape(-1)
def xpolicylab_action_to_robotwin(
action: Any,
*,
action_type: str,
current_observation=None,
):
robotwin_action_type = normalize_robotwin_action_type(action_type)
if not isinstance(action, Mapping):
return np.asarray(action, dtype=np.float32).reshape(-1), robotwin_action_type
resolved_action_type = action.get("action_type", action_type)
robotwin_action_type = normalize_robotwin_action_type(str(resolved_action_type))
if robotwin_action_type == "qpos":
left_arm = action_array(
action,
("left_arm_joint_state", "left_arm_joint", "left_joint_state", "arm_joint_state", "joint_state"),
fallback=current_joint_arm(current_observation, "left"),
name="left arm joint action",
)
right_arm = action_array(
action,
("right_arm_joint_state", "right_arm_joint", "right_joint_state"),
fallback=current_joint_arm(current_observation, "right"),
name="right arm joint action",
)
left_gripper = action_gripper(
action,
("left_ee_joint_state", "left_gripper", "left_gripper_pos", "ee_joint_state"),
fallback=current_gripper(current_observation, "left"),
name="left gripper action",
)
right_gripper = action_gripper(
action,
("right_ee_joint_state", "right_gripper", "right_gripper_pos"),
fallback=current_gripper(current_observation, "right"),
name="right gripper action",
)
else:
left_arm = action_array(
action,
("left_ee_pose", "left_endpose", "ee_pose"),
fallback=current_ee_pose(current_observation, "left"),
name="left ee action",
expected_dim=7,
)
right_arm = action_array(
action,
("right_ee_pose", "right_endpose"),
fallback=current_ee_pose(current_observation, "right"),
name="right ee action",
expected_dim=7,
)
left_gripper = action_gripper(
action,
("left_ee_joint_state", "left_gripper", "left_gripper_pos", "ee_joint_state"),
fallback=current_gripper(current_observation, "left"),
name="left gripper action",
)
right_gripper = action_gripper(
action,
("right_ee_joint_state", "right_gripper", "right_gripper_pos"),
fallback=current_gripper(current_observation, "right"),
name="right gripper action",
)
flat_action = np.concatenate(
[
left_arm.astype(np.float32),
np.asarray([left_gripper], dtype=np.float32),
right_arm.astype(np.float32),
np.asarray([right_gripper], dtype=np.float32),
]
)
return flat_action, robotwin_action_type
# ============================================================
# 单 episode 评测 (来源: run_one_batch_episode, 单进程化)
# ============================================================
def _load_official_step_limit(task_name: str) -> int:
"""官方 _eval_step_limit.yml 的每任务步数上限 (缺省 400)。"""
import yaml
path = Path(CONFIGS_PATH) / "_eval_step_limit.yml"
try:
with open(path, encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
return int(data.get(task_name, 400))
except (OSError, TypeError, ValueError):
return 400
def run_episode(
episode_id: int,
seed_value: int,
task_name: str,
task_env,
args: dict[str, Any],
model_client,
hp: dict,
instruction_type: str | None,
video_size: str | None,
) -> dict[str, Any]:
action_type = hp["action_type"]
frequency = hp["frequency"]
expert_check = hp["expert_check"]
render_freq = args["render_freq"]
args["render_freq"] = 0
episode_info = {"info": {}}
if expert_check:
try:
task_env.setup_demo(now_ep_num=episode_id, seed=seed_value, is_test=True, **args)
episode_info = task_env.play_once()
task_env.close_env()
except UnStableError:
task_env.close_env()
args["render_freq"] = render_freq
return {"type": "seed_skipped", "seed": seed_value, "reason": "unstable"}
except Exception as e:
task_env.close_env()
args["render_freq"] = render_freq
print(f"error occurs during expert check! seed={seed_value} err={type(e).__name__}: {e}")
return {"type": "seed_skipped", "seed": seed_value, "reason": "expert_error"}
if expert_check and not (task_env.plan_success and task_env.check_success()):
args["render_freq"] = render_freq
return {"type": "seed_skipped", "seed": seed_value, "reason": "expert_failed"}
args["render_freq"] = render_freq
try:
task_env.setup_demo(now_ep_num=episode_id, seed=seed_value, is_test=True, **args)
except UnStableError:
try:
task_env.close_env()
except Exception:
pass
print(f"skip unstable seed={seed_value} (eval setup)")
return {"type": "seed_skipped", "seed": seed_value, "reason": "unstable"}
# 改动点 4: STEP_LIMIT 绝对值覆盖 (is_episode_end 与 take_action 双消费点同时生效);
# 空值时回退官方 _eval_step_limit.yml 每任务值 (任务 env 默认 step_lim=None,
# 缺失时 is_episode_end 的 >= 比较直接 TypeError, 改动点 4 声明的回退此前未实现)
if hp["step_limit"] > 0:
task_env.step_lim = int(hp["step_limit"])
elif getattr(task_env, "step_lim", None) is None:
task_env.step_lim = _load_official_step_limit(task_name)
instruction = build_instruction(args, episode_info, instruction_type, hp["test_num"])
task_env.set_instruction(instruction=instruction)
if getattr(task_env, "eval_video_path", None) is not None:
ffmpeg = subprocess.Popen(
[
"ffmpeg",
"-y",
"-loglevel", "error",
"-f", "rawvideo",
"-pixel_format", "rgb24",
"-video_size", video_size,
"-framerate", "10",
"-i", "-",
"-pix_fmt", "yuv420p",
"-vcodec", "libx264",
"-crf", "23",
f"{task_env.eval_video_path}/episode{episode_id}.mp4",
],
stdin=subprocess.PIPE,
)
task_env._set_eval_video_ffmpeg(ffmpeg)
succ = False
prepare_policy_case(model_client, task_name, seed_value, instruction, action_type)
reset_policy(model_client)
try:
while not is_episode_end(task_env):
observation = task_env.get_obs()
xpl_obs = robotwin_obs_to_xpolicylab(
observation,
instruction=task_env.get_instruction(),
env_idx=0,
frequency=frequency,
task_env=task_env,
)
model_client.call(func_name="update_obs", obs=xpl_obs)
action_chunk = normalize_action_chunk(model_client.call(func_name="get_action"))
if len(action_chunk) == 0:
raise RuntimeError("Policy returned an empty action chunk.")
# ACTION_STRIDE: 每 chunk 只执行前 N 个动作后即重新请求动作块 (0 = 整块执行)
n_exec = len(action_chunk)
if hp["action_stride"] > 0:
n_exec = min(hp["action_stride"], len(action_chunk))
for action_idx, action in enumerate(action_chunk):
flat_action, robotwin_action_type = xpolicylab_action_to_robotwin(
action,
action_type=action_type,
current_observation=observation,
)
task_env.take_action(flat_action, action_type=robotwin_action_type)
if task_env.eval_success:
succ = True
break
if is_episode_end(task_env) or action_idx + 1 >= n_exec:
break
# 块内不逐动作发送 update_obs: 外层循环每次 get_action 前都会重新取 obs
# 并 update, 块内逐动作发送属纯冗余; 一并省去块内逐动作的相机渲染与
# 网络带宽 (远程策略服务场景带宽降幅显著)。块内动作转换 fallback 使用
# 块首 obs, 数组动作不受影响。
if succ:
break
except Exception:
print(f"\n\033[91mPolicy rollout error (task {task_name}, seed {seed_value}):\033[0m")
print(traceback.format_exc())
if getattr(task_env, "eval_video_path", None) is not None:
task_env._del_eval_video_ffmpeg()
notify_trial_end(model_client, task_name, seed_value, succ)
task_env.close_env()
if task_env.render_freq:
task_env.viewer.close()
return {"type": "episode_done", "episode_id": episode_id, "seed": seed_value, "success": bool(succ)}
def eval_one_task(task_name: str, hp: dict, model_client) -> tuple[int, int]:
args = load_task_args(task_name, hp)
instruction_type = resolve_eval_instruction(hp, args)
args["eval_instruction"] = instruction_type
output_dir = Path(hp["output_dir"])
task_result_dir = output_dir / "eval_result" / task_name
task_result_dir.mkdir(parents=True, exist_ok=True)
# 改动点 5: 回灌视频目录 $OUTPUT_DIR/videos/<task>/
video_size = None
if args["eval_video_log"]:
camera_config = get_camera_config(args["camera"]["head_camera_type"])
video_size = str(camera_config["w"]) + "x" + str(camera_config["h"])
args["eval_video_save_dir"] = str(output_dir / "videos" / task_name)
(output_dir / "videos" / task_name).mkdir(parents=True, exist_ok=True)
print(f"\n\033[96m===== Eval Task: {task_name} (instruction={instruction_type}) =====\033[0m")
task_env = class_decorator(task_name)
seed = int(hp["seed"])
st_seed = 100000 * (1 + seed)
test_num = int(hp["test_num"])
max_seed_attempts = int(hp["max_seed_attempts"]) or (test_num * 50)
episode_id, success_num, completed, skipped = 0, 0, 0, 0
for seed_value in range(st_seed, st_seed + max_seed_attempts):
if completed >= test_num:
break
result = run_episode(
episode_id, seed_value, task_name, task_env, args, model_client, hp,
instruction_type, video_size,
)
if result["type"] == "seed_skipped":
skipped += 1
continue
episode_id += 1
completed += 1
success_num += int(bool(result["success"]))
print(
f"\033[93m{task_name}\033[0m | episode={result['episode_id']} | seed={result['seed']} | "
f"success={result['success']} | task success: \033[96m{success_num}/{completed}\033[0m"
)
report_utils.emit_progress(
f"eval:{task_name}",
episode=completed,
maxEpisode=test_num,
success=success_num,
)
try:
task_env.close_env()
except Exception:
pass
if completed == 0:
raise RuntimeError(
f"Eval completed zero episodes for task {task_name}. skipped_seeds={skipped}, "
f"max_seed_attempts={max_seed_attempts}"
)
with open(task_result_dir / "_result.txt", "w", encoding="utf-8") as file:
file.write(f"Instruction Type: {instruction_type}\n")
file.write(f"Skipped Seeds: {skipped}\n")
file.write(f"{success_num}/{completed}\n")
return success_num, completed
# ============================================================
# Main: 多任务循环 + 汇总 (改动点 1/7)
# ============================================================
def main() -> None:
hp = read_hyperparams()
np.random.seed(int(hp["seed"])) # 指令随机选择可复现
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S")
tasks_summary: dict[str, str] = {}
total_success, total_completed = 0, 0
model_client = build_policy_client(hp)
try:
for task_name in hp["task_names"]:
suc, done = eval_one_task(task_name, hp, model_client)
tasks_summary[task_name] = f"{suc}/{done}"
total_success += suc
total_completed += done
finally:
close_policy_client(model_client)
overall = round(total_success / total_completed, 4) if total_completed else 0.0
summary = {
"tasks": tasks_summary,
"overall": overall,
"config": {
"timestamp": current_time,
"policy_server": f"{hp['policy_server_ip']}:{hp['policy_server_port']}",
"protocol": hp["protocol"],
"policy_name": hp["policy_name"],
"ckpt_setting": hp["ckpt_setting"],
"task_config": hp["task_config"],
"test_num": hp["test_num"],
"seed": hp["seed"],
"action_type": hp["action_type"],
"frequency": hp["frequency"],
},
}
output_dir = Path(hp["output_dir"])
(output_dir / "eval_summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8"
)
print(f"Overall success rate: {total_success}/{total_completed} = {round(overall * 100, 1)}%")
report_utils.write_artifacts(
output_dir,
[
{"name": "eval_result", "type": "file", "portId": "eval_result", "path": str(output_dir / "eval_result")},
{"name": "eval_summary", "type": "json", "portId": "eval_report", "path": str(output_dir / "eval_summary.json")},
],
)
if __name__ == "__main__":
sapien_render_check()
main()