fix: enable production headless rollout rendering

This commit is contained in:
eust-w 2026-07-11 05:08:30 +08:00
parent ed19659eef
commit a64fdf4836
5 changed files with 97 additions and 11 deletions

View File

@ -182,6 +182,10 @@ spec:
- {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc}
- {name: REAL2SIM_RUNTIME_GATE_DIR, value: /var/lib/real2sim-poc/cache/runtime-gate}
- {name: REAL2SIM_MOTRIXSIM_PYTHON, value: /opt/conda/bin/python}
- {name: REAL2SIM_FFMPEG_BIN, value: /usr/bin/ffmpeg}
- {name: MUJOCO_GL, value: egl}
- {name: PYOPENGL_PLATFORM, value: egl}
- {name: NVIDIA_DRIVER_CAPABILITIES, value: "compute,utility,graphics"}
- {name: HF_HOME, value: /var/lib/real2sim-poc/cache/huggingface}
- {name: HF_HUB_DISABLE_XET, value: "1"}
- {name: HOME, value: /var/lib/real2sim-poc/cache/home}

View File

@ -73,7 +73,11 @@ RUN python -c 'import mujoco, motrixsim, torch; from huggingface_hub import snap
ENV REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \
REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT} \
REAL2SIM_MOTRIXSIM_DOCS_COMMIT=${MOTRIXSIM_DOCS_COMMIT} \
REAL2SIM_MOTRIXSIM_PYTHON=/opt/conda/bin/python
REAL2SIM_MOTRIXSIM_PYTHON=/opt/conda/bin/python \
REAL2SIM_FFMPEG_BIN=/usr/bin/ffmpeg \
MUJOCO_GL=egl \
PYOPENGL_PLATFORM=egl \
NVIDIA_DRIVER_CAPABILITIES=compute,utility,graphics
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim && \
chown -R 10001:10001 /app

View File

@ -3,6 +3,8 @@ from __future__ import annotations
import argparse
import json
import math
import os
import shutil
import subprocess
import time
from dataclasses import dataclass
@ -298,8 +300,9 @@ def _write_preview_video(path: Path, images: list[np.ndarray]) -> None:
if not images:
raise ValueError("cannot encode an empty rollout")
height, width = images[0].shape[:2]
ffmpeg = os.environ.get("REAL2SIM_FFMPEG_BIN", "").strip() or shutil.which("ffmpeg") or "ffmpeg"
command = [
"ffmpeg",
ffmpeg,
"-y",
"-loglevel",
"error",
@ -320,14 +323,25 @@ def _write_preview_video(path: Path, images: list[np.ndarray]) -> None:
"yuv420p",
str(path),
]
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
try:
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
except OSError as exc:
raise RuntimeError(f"ffmpeg rollout encoder could not start {ffmpeg}: {exc}") from exc
assert process.stdin is not None
for image in images:
process.stdin.write(np.ascontiguousarray(image, dtype=np.uint8).tobytes())
process.stdin.close()
write_error: BrokenPipeError | None = None
try:
for image in images:
process.stdin.write(np.ascontiguousarray(image, dtype=np.uint8).tobytes())
except BrokenPipeError as exc:
write_error = exc
try:
process.stdin.close()
except BrokenPipeError as exc:
write_error = write_error or exc
stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else ""
if process.wait() != 0 or not path.is_file() or path.stat().st_size == 0:
raise RuntimeError(f"ffmpeg rollout encoding failed: {stderr[-2000:]}")
return_code = process.wait()
if write_error or return_code != 0 or not path.is_file() or path.stat().st_size == 0:
raise RuntimeError(f"ffmpeg rollout encoding failed with {ffmpeg} (exit {return_code}): {stderr[-2000:]}") from write_error
def _damped_least_squares(jacobian: np.ndarray, error: np.ndarray, damping: float) -> np.ndarray:

View File

@ -5,6 +5,7 @@ import gc
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
@ -133,13 +134,25 @@ def _nvidia_probe() -> dict[str, Any]:
def _simulator_probe() -> dict[str, Any]:
xml = """<mujoco model="runtime-gate"><option timestep="0.002"/><worldbody><body pos="0 0 0.2"><freejoint/><geom type="sphere" size="0.03" mass="0.1"/></body><geom type="plane" size="1 1 .1"/></worldbody></mujoco>"""
xml = """<mujoco model="runtime-gate"><option timestep="0.002"/><worldbody><geom type="plane" size="2 2 .1" rgba=".3 .3 .3 1"/><body pos=".65 0 .2"><freejoint/><geom type="box" size=".1 .1 .1" rgba="1 0 0 1" mass=".1"/></body><camera name="overview" pos="1.7 -1.6 1.25" xyaxes="0.68 0.73 0 -0.37 0.34 0.86"/></worldbody></mujoco>"""
import mujoco
import numpy as np
model = mujoco.MjModel.from_xml_string(xml)
data = mujoco.MjData(model)
for _ in range(8):
mujoco.mj_step(model, data)
renderer = mujoco.Renderer(model, height=120, width=160)
try:
renderer.update_scene(data, camera="overview")
image = renderer.render()
finally:
renderer.close()
image_mean = float(image.mean())
image_std = float(image.std())
nonzero_pixels = int(np.count_nonzero(image))
if nonzero_pixels <= 0 or image_std <= 0:
raise RuntimeError(f"MuJoCo EGL renderer produced a blank image: mean={image_mean}, std={image_std}")
with tempfile.TemporaryDirectory(prefix="real2sim-runtime-gate-") as temporary:
scene_path = Path(temporary) / "scene.xml"
scene_path.write_text(xml, encoding="utf-8")
@ -149,7 +162,48 @@ def _simulator_probe() -> dict[str, Any]:
motrix_data = SceneData(motrix_model)
for _ in range(8):
step(motrix_model, motrix_data)
return {"ok": True, "mujoco_time": float(data.time), "motrix_timestep": float(motrix_model.options.timestep)}
ffmpeg = os.environ.get("REAL2SIM_FFMPEG_BIN", "").strip() or shutil.which("ffmpeg")
if not ffmpeg:
raise RuntimeError("ffmpeg executable is unavailable")
video_path = Path(temporary) / "probe.mp4"
completed = subprocess.run(
[
ffmpeg,
"-y",
"-loglevel",
"error",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
"-s",
"160x120",
"-r",
"20",
"-i",
"-",
"-an",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
str(video_path),
],
input=image.tobytes() * 2,
capture_output=True,
timeout=30,
check=False,
)
if completed.returncode != 0 or not video_path.is_file() or video_path.stat().st_size == 0:
raise RuntimeError(f"ffmpeg H.264 probe failed with {ffmpeg}: {completed.stderr.decode('utf-8', errors='replace')[-1000:]}")
video_size = video_path.stat().st_size
return {
"ok": True,
"mujoco_time": float(data.time),
"motrix_timestep": float(motrix_model.options.timestep),
"renderer": {"shape": list(image.shape), "mean": image_mean, "std": image_std, "nonzero_pixels": nonzero_pixels},
"ffmpeg": {"path": ffmpeg, "video_size": video_size, "codec": "libx264"},
}
def _motrixsim_assets_probe() -> dict[str, Any]:

View File

@ -6,12 +6,13 @@ import sys
from pathlib import Path
import h5py
import numpy as np
import pytest
from fastapi.testclient import TestClient
from real2sim_platform.config import PlatformConfig
from real2sim_platform.adapters import index_artifacts_for_task
from real2sim_platform.rm65_controller import collect
from real2sim_platform.rm65_controller import _write_preview_video, collect
from real2sim_platform.runtime_gate import _motrixsim_assets_probe, runtime_key
from real2sim_platform.server import create_app
from real2sim_platform.storage import BosStorage, StorageBoundaryError, normalize_relative_path
@ -277,9 +278,18 @@ def test_worker_image_materializes_motrixsim_lfs_assets() -> None:
assert "git-lfs.github.com/spec" in dockerfile
manifest = (ROOT / "deploy/poc/app-template.yaml").read_text(encoding="utf-8")
assert "REAL2SIM_MOTRIXSIM_PYTHON" in manifest
assert "REAL2SIM_FFMPEG_BIN" in manifest
assert "NVIDIA_DRIVER_CAPABILITIES" in manifest
assert "compute,utility,graphics" in manifest
assert "GIT_CONFIG_VALUE_3, value: /opt/motrixsim-docs" in manifest
def test_rollout_video_encoder_reports_the_selected_ffmpeg_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("REAL2SIM_FFMPEG_BIN", "/usr/bin/false")
with pytest.raises(RuntimeError, match="ffmpeg rollout encoding failed with /usr/bin/false"):
_write_preview_video(tmp_path / "rollout.mp4", [np.zeros((16, 16, 3), dtype=np.uint8)])
def test_runtime_gate_rejects_unresolved_motrixsim_lfs_pointer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
root = tmp_path / "motrixsim-docs"
required = [