290 lines
11 KiB
Python
290 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gc
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from huggingface_hub import snapshot_download
|
|
|
|
from .adapters import run_doctor
|
|
from .config import PlatformConfig, ensure_workspace, load_config
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Validate the production Real2Sim GPU runtime")
|
|
parser.add_argument("--mode", choices=["bootstrap", "readiness"], default="readiness")
|
|
parser.add_argument("--force", action="store_true")
|
|
args = parser.parse_args()
|
|
config = load_config()
|
|
ensure_workspace(config)
|
|
marker = marker_path(config)
|
|
if args.mode == "readiness":
|
|
return readiness(config, marker)
|
|
if marker.is_file() and not args.force:
|
|
try:
|
|
existing = json.loads(marker.read_text(encoding="utf-8"))
|
|
if existing.get("runtime_key") == runtime_key(config) and existing.get("ok") is True:
|
|
print(json.dumps(existing, ensure_ascii=False))
|
|
return 0
|
|
except (OSError, json.JSONDecodeError):
|
|
pass
|
|
report = bootstrap(config)
|
|
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = marker.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(marker)
|
|
print(json.dumps(report, ensure_ascii=False))
|
|
return 0 if report["ok"] else 1
|
|
|
|
|
|
def marker_path(config: PlatformConfig) -> Path:
|
|
root = Path(os.environ.get("REAL2SIM_RUNTIME_GATE_DIR", config.workspace_dir / "runtime_gate"))
|
|
return root / f"{runtime_key(config)}.json"
|
|
|
|
|
|
def runtime_key(config: PlatformConfig) -> str:
|
|
raw = ":".join(
|
|
[
|
|
config.anysplat_commit,
|
|
config.anysplat_model_revision,
|
|
config.vggt_model_revision,
|
|
config.discoverse_commit,
|
|
config.act_commit,
|
|
os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_COMMIT", ""),
|
|
os.environ.get("REAL2SIM_PLATFORM_BUILD_ID", ""),
|
|
]
|
|
)
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:20]
|
|
|
|
|
|
def readiness(config: PlatformConfig, marker: Path) -> int:
|
|
doctor = run_doctor(config)
|
|
if not marker.is_file():
|
|
print("runtime bootstrap report is missing", file=sys.stderr)
|
|
return 1
|
|
try:
|
|
report = json.loads(marker.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
print(f"runtime bootstrap report is unreadable: {exc}", file=sys.stderr)
|
|
return 1
|
|
ready = bool(report.get("ok") and report.get("runtime_key") == runtime_key(config) and doctor.get("ok"))
|
|
if not ready:
|
|
print(json.dumps({"bootstrap": report, "doctor": doctor}, ensure_ascii=False), file=sys.stderr)
|
|
return 0 if ready else 1
|
|
|
|
|
|
def bootstrap(config: PlatformConfig) -> dict[str, Any]:
|
|
started = time.time()
|
|
checks: dict[str, Any] = {}
|
|
doctor = run_doctor(config)
|
|
checks["doctor"] = {"ok": doctor.get("ok"), "core_checks": doctor.get("core_checks"), "checks": doctor.get("checks")}
|
|
try:
|
|
checks["nvidia"] = _nvidia_probe()
|
|
checks["simulators"] = _simulator_probe()
|
|
checks["motrixsim_assets"] = _motrixsim_assets_probe()
|
|
checks["act"] = _act_probe(config)
|
|
checks["anysplat"] = _anysplat_probe(config)
|
|
except Exception as exc:
|
|
checks["failure"] = {"type": type(exc).__name__, "message": str(exc)}
|
|
ok = bool(doctor.get("ok")) and all(bool(value.get("ok")) for key, value in checks.items() if key != "failure") and "failure" not in checks
|
|
return {
|
|
"schema": "real2sim.runtime_gate.v1",
|
|
"runtime_key": runtime_key(config),
|
|
"ok": ok,
|
|
"started_at": started,
|
|
"finished_at": time.time(),
|
|
"checks": checks,
|
|
"revisions": {
|
|
"anysplat_code": config.anysplat_commit,
|
|
"anysplat_model": config.anysplat_model_revision,
|
|
"vggt_model": config.vggt_model_revision,
|
|
"discoverse": config.discoverse_commit,
|
|
"act": config.act_commit,
|
|
"motrixsim_docs": os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_COMMIT", ""),
|
|
},
|
|
}
|
|
|
|
|
|
def _nvidia_probe() -> dict[str, Any]:
|
|
completed = subprocess.run(
|
|
["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=20,
|
|
check=True,
|
|
)
|
|
import torch
|
|
|
|
if not torch.cuda.is_available():
|
|
raise RuntimeError("PyTorch CUDA is unavailable")
|
|
value = torch.ones((1024, 1024), device="cuda")
|
|
checksum = float((value @ value).mean().item())
|
|
del value
|
|
torch.cuda.empty_cache()
|
|
return {"ok": checksum == 1024.0, "device": torch.cuda.get_device_name(0), "nvidia_smi": completed.stdout.strip().splitlines()[0]}
|
|
|
|
|
|
def _simulator_probe() -> dict[str, Any]:
|
|
xml = """<mujoco model="runtime-gate"><option timestep="0.002"/><worldbody><geom type="plane" size="2 2 .1" rgba=".3 .3 .3 1"/><body pos=".65 0 .2"><freejoint/><geom type="box" size=".1 .1 .1" rgba="1 0 0 1" mass=".1"/></body><camera name="overview" pos="1.7 -1.6 1.25" xyaxes="0.68 0.73 0 -0.37 0.34 0.86"/></worldbody></mujoco>"""
|
|
import mujoco
|
|
import numpy as np
|
|
|
|
model = mujoco.MjModel.from_xml_string(xml)
|
|
data = mujoco.MjData(model)
|
|
for _ in range(8):
|
|
mujoco.mj_step(model, data)
|
|
renderer = mujoco.Renderer(model, height=120, width=160)
|
|
try:
|
|
renderer.update_scene(data, camera="overview")
|
|
image = renderer.render()
|
|
finally:
|
|
renderer.close()
|
|
image_mean = float(image.mean())
|
|
image_std = float(image.std())
|
|
nonzero_pixels = int(np.count_nonzero(image))
|
|
if nonzero_pixels <= 0 or image_std <= 0:
|
|
raise RuntimeError(f"MuJoCo EGL renderer produced a blank image: mean={image_mean}, std={image_std}")
|
|
with tempfile.TemporaryDirectory(prefix="real2sim-runtime-gate-") as temporary:
|
|
scene_path = Path(temporary) / "scene.xml"
|
|
scene_path.write_text(xml, encoding="utf-8")
|
|
from motrixsim import SceneData, load_model, step
|
|
|
|
motrix_model = load_model(str(scene_path))
|
|
motrix_data = SceneData(motrix_model)
|
|
for _ in range(8):
|
|
step(motrix_model, motrix_data)
|
|
ffmpeg = os.environ.get("REAL2SIM_FFMPEG_BIN", "").strip() or shutil.which("ffmpeg")
|
|
if not ffmpeg:
|
|
raise RuntimeError("ffmpeg executable is unavailable")
|
|
video_path = Path(temporary) / "probe.mp4"
|
|
completed = subprocess.run(
|
|
[
|
|
ffmpeg,
|
|
"-y",
|
|
"-loglevel",
|
|
"error",
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
"-s",
|
|
"160x120",
|
|
"-r",
|
|
"20",
|
|
"-i",
|
|
"-",
|
|
"-an",
|
|
"-c:v",
|
|
"libx264",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
str(video_path),
|
|
],
|
|
input=image.tobytes() * 2,
|
|
capture_output=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
if completed.returncode != 0 or not video_path.is_file() or video_path.stat().st_size == 0:
|
|
raise RuntimeError(f"ffmpeg H.264 probe failed with {ffmpeg}: {completed.stderr.decode('utf-8', errors='replace')[-1000:]}")
|
|
video_size = video_path.stat().st_size
|
|
return {
|
|
"ok": True,
|
|
"mujoco_time": float(data.time),
|
|
"motrix_timestep": float(motrix_model.options.timestep),
|
|
"renderer": {"shape": list(image.shape), "mean": image_mean, "std": image_std, "nonzero_pixels": nonzero_pixels},
|
|
"ffmpeg": {"path": ffmpeg, "video_size": video_size, "codec": "libx264"},
|
|
}
|
|
|
|
|
|
def _motrixsim_assets_probe() -> dict[str, Any]:
|
|
root = Path(os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_ROOT", "/opt/motrixsim-docs"))
|
|
required = {
|
|
"parlour": root / "examples/assets/parlour/parlour.xml",
|
|
"rm65_mesh": root / "examples/assets/rm65_open_cabinet/assets/rm65/meshes/base_link.STL",
|
|
"pick_place_template": root / "examples/assets/franka_emika_panda/scene_pick_cube.xml",
|
|
}
|
|
missing = [str(path) for path in required.values() if not path.is_file()]
|
|
if missing:
|
|
raise RuntimeError("MotrixSim assets are missing: " + ", ".join(missing))
|
|
pointers = []
|
|
for name, path in required.items():
|
|
if path.stat().st_size <= 1024 and path.read_bytes()[:160].startswith(b"version https://git-lfs.github.com/spec/"):
|
|
pointers.append(name)
|
|
if pointers:
|
|
raise RuntimeError("MotrixSim Git LFS assets are unresolved: " + ", ".join(pointers))
|
|
completed = subprocess.run(
|
|
["git", "-C", str(root), "rev-parse", "HEAD"],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=20,
|
|
check=True,
|
|
)
|
|
commit = completed.stdout.strip()
|
|
expected = os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_COMMIT", "")
|
|
if expected and commit != expected:
|
|
raise RuntimeError(f"MotrixSim docs revision mismatch: expected {expected}, got {commit}")
|
|
return {"ok": True, "commit": commit, "asset_sizes": {name: path.stat().st_size for name, path in required.items()}}
|
|
|
|
|
|
def _act_probe(config: PlatformConfig) -> dict[str, Any]:
|
|
act_root = config.discoverse_root / "policies" / "act"
|
|
env = os.environ.copy()
|
|
env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(config.discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep)
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-c",
|
|
"import torch; import policy_train; import utils.utils; from policies.act.act import ACTPolicy; "
|
|
"print(torch.__version__); assert ACTPolicy is not None and torch.cuda.is_available()",
|
|
],
|
|
cwd=act_root,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
check=True,
|
|
)
|
|
return {"ok": True, "torch": completed.stdout.strip().splitlines()[-1], "training_stack_imported": True}
|
|
|
|
|
|
def _anysplat_probe(config: PlatformConfig) -> dict[str, Any]:
|
|
cache_dir = Path(os.environ.get("HF_HOME", config.workspace_dir / "cache" / "huggingface"))
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
anysplat_snapshot = Path(
|
|
snapshot_download(
|
|
repo_id="lhjiang/anysplat",
|
|
revision=config.anysplat_model_revision,
|
|
cache_dir=cache_dir,
|
|
)
|
|
)
|
|
snapshot_download(repo_id="facebook/VGGT-1B", revision=config.vggt_model_revision, cache_dir=cache_dir)
|
|
if str(config.anysplat_root) not in sys.path:
|
|
sys.path.insert(0, str(config.anysplat_root))
|
|
from src.model.model.anysplat import AnySplat
|
|
|
|
model = AnySplat.from_pretrained(str(anysplat_snapshot))
|
|
model = model.to("cuda").eval()
|
|
parameters = sum(parameter.numel() for parameter in model.parameters())
|
|
if parameters <= 0:
|
|
raise RuntimeError("AnySplat model loaded with zero parameters")
|
|
del model
|
|
gc.collect()
|
|
import torch
|
|
|
|
torch.cuda.empty_cache()
|
|
return {"ok": True, "parameter_count": parameters, "snapshot": anysplat_snapshot.name}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|