feat: add MotrixSim scene composition workbench

This commit is contained in:
eust-w 2026-07-12 16:48:15 +08:00
parent 198661f3e4
commit dd7312d3dd
15 changed files with 1878 additions and 37 deletions

View File

@ -26,6 +26,7 @@ data:
anysplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python"
cuda_home = "/usr/local/cuda"
runtime_env_script = "/opt/gs_playground/scripts/real2sim_wuying_paper_env.sh"
motrixsim_docs_root = "/opt/motrixsim-docs"
allowed_path_roots = "/var/lib/real2sim-poc:/opt/DISCOVERSE:/opt/gs_playground:/opt/motrixsim-docs:/opt/real2sim_paper_runtime:/tmp"
production_mode = true
auth_enabled = true

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 MiB

View File

@ -42,6 +42,7 @@ where = ["src"]
"static/vendor/three/addons/loaders/*",
"static/vendor/three/addons/postprocessing/*",
"static/vendor/three/addons/utils/*",
"static/assets/robots/*",
]
[tool.pytest.ini_options]

View File

@ -0,0 +1,22 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from pathlib import Path
from real2sim_platform.motrix_scene import export_mujoco_body_glb
def main() -> int:
parser = argparse.ArgumentParser(description="Export the registered RM65 MJCF visual geometry as a browser GLB")
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = export_mujoco_body_glb(args.source.resolve(), args.output.resolve(), root_body="base_link", relative_to_root=True)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -17,7 +17,7 @@ from urllib.parse import quote, unquote, urlparse
from urllib.request import urlopen
from .config import PlatformConfig, is_path_allowed
from .motrix_scene import build_spatial_scene_package, compose_scene_asset
from .motrix_scene import COMPOSITION_ASSET_RECIPES, build_scene_composition, build_spatial_scene_package, compose_scene_asset
from .source_revision import read_source_revision
from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path
from .store import PlatformStore
@ -194,6 +194,8 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A
return run_motrixsim_scene_import(config, store, task, emit)
if task_type == "scene_asset_compose":
return run_scene_asset_compose(config, store, task, emit)
if task_type == "scene_composition":
return run_scene_composition(config, store, task, emit)
if task_type == "simulator_validation":
return run_simulator_validation(config, store, task, emit)
if task_type == "expert_collection":
@ -881,6 +883,104 @@ def run_scene_asset_compose(config: PlatformConfig, store: PlatformStore, task:
}
def run_scene_composition(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
params = task["params"]
parent = store.get_scene(str(params["scene_id"]))
source_package = Path(str((parent.get("metadata") or {}).get("package") or ""))
if not source_package.is_dir():
scene_path = Path(parent["path"])
source_package = scene_path if scene_path.is_dir() else scene_path.parent
if not (source_package / "scene_manifest.json").is_file():
raise ValueError("selected scene is not an editable MotrixSim scene package")
run_dir = Path(task["run_dir"])
composition_dir = run_dir / "scene_composition"
_task_progress(store, task, 12, "创建组合版本", "正在复制基础场景,原场景和原始重建产物保持不变")
_copy_tree_isolated(source_package, composition_dir)
_rebase_package_metadata(composition_dir, source_package)
composition = params.get("composition") or {}
asset_specs = list(composition.get("assets") or [])
source_paths: dict[str, Path] = {}
for index, spec in enumerate(asset_specs):
catalog_id = str(spec.get("catalog_id") or "")
if catalog_id in COMPOSITION_ASSET_RECIPES or catalog_id.startswith("primitive:"):
continue
instance_id = str(spec.get("instance_id") or f"asset_{index + 1}")
source_paths[instance_id] = _resolve_asset_source(config, store, spec.get("source") or {}, spec.get("path"))
robot_source = config.motrixsim_docs_root / "examples" / "assets" / "rm65_open_cabinet" / "mjx_rm65.xml"
_task_progress(store, task, 28, "合并 RM65", "正在写入真实关节、执行器、夹爪碰撞体和 MotrixSim 资产")
result = build_scene_composition(
composition_dir,
robot_source=robot_source,
robot_spec=composition.get("robot") or {},
asset_specs=asset_specs,
source_paths=source_paths,
validation_steps=int(params.get("validation_steps", 120)),
emit=emit,
)
_task_progress(store, task, 82, "登记组合场景", "双运行时校验通过,正在登记预览资产、控制器和场景血缘")
artifacts = _index_artifacts(store, task["id"], composition_dir)
manifest_path = Path(result["manifest_path"])
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
revision_id = uuid.uuid4().hex
manifest["revision"] = {
"id": revision_id,
"parent_scene_id": parent["id"],
"reason": "motrixsim_scene_composition",
"task_id": task["id"],
}
_attach_artifact_urls(manifest, artifacts)
manifest_path.write_text(json.dumps(enrich_scene_manifest_assets(manifest), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
validation = result["validation"]
metadata = dict(parent.get("metadata") or {})
metadata.update(
{
"package": str(composition_dir),
"parent_scene_id": parent["id"],
"revision_reason": "motrixsim_scene_composition",
"runtime_validation": {"mujoco": bool(validation["mujoco"]["ok"]), "motrixsim": bool(validation["motrixsim"]["ok"])},
"editable": True,
"robot_id": "rm65",
"controller": "rm65_pick_place_6d_ik" if result["collection_ready"] else None,
"collection_ready": bool(result["collection_ready"]),
}
)
name = str(params.get("name") or composition.get("name") or f"{parent['name']} · RM65 组合")
scene = store.create_scene(
item_id=revision_id,
name=name,
path=Path(result["runtime_scene"]),
metadata=metadata,
source_type="scene_composition",
source_uri=parent.get("storage_uri") or parent.get("source_uri") or parent["path"],
version=_next_scene_revision(str(parent.get("version") or "v1")),
parent_task_id=task["id"],
manifest_path=manifest_path,
metrics={
**(parent.get("metrics") or {}),
"composition_asset_count": len(asset_specs),
"robot_count": 1,
"collection_ready": bool(result["collection_ready"]),
},
tags=list(dict.fromkeys([*(parent.get("tags") or []), "scene-composition", "motrixsim", "rm65", "runtime-validated", *( ["collection-ready"] if result["collection_ready"] else [])])),
cache_state="local",
)
_task_progress(store, task, 96, "组合场景可用", "组合场景已可测试;满足控制器对象约定时可直接进入专家采集")
return {
"scene_id": scene["id"],
"parent_scene_id": parent["id"],
"scene_path": scene["path"],
"manifest_path": str(manifest_path),
"composition": result["composition"],
"collection_ready": bool(result["collection_ready"]),
"runtime_validation": validation,
"report_path": result["report_path"],
}
def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
params = task["params"]
run_dir = Path(task["run_dir"])

View File

@ -32,6 +32,7 @@ class PlatformConfig:
anysplat_python: Path = Path("/opt/real2sim_paper_runtime/envs/anysplat/bin/python")
cuda_home: Path = Path("/usr/local/cuda-12.1")
runtime_env_script: Path = Path("/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh")
motrixsim_docs_root: Path = Path("/opt/motrixsim-docs")
allowed_path_roots: tuple[Path, ...] = ()
worker_mode: str = "embedded"
worker_id: str = "local-worker"
@ -99,6 +100,7 @@ class PlatformConfig:
self.discoverse_root,
self.runtime_root,
self.anysplat_root,
self.motrixsim_docs_root,
Path("/opt/gs_playground_real2sim_48q"),
Path("/tmp"),
)
@ -129,6 +131,7 @@ def load_config(path: Path | None = None) -> PlatformConfig:
anysplat_python=Path(str(values.get("anysplat_python", values.get("gsplat_python", "/opt/real2sim_paper_runtime/envs/anysplat/bin/python")))),
cuda_home=Path(str(values.get("cuda_home", "/usr/local/cuda-12.1"))),
runtime_env_script=Path(str(values.get("runtime_env_script", "/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh"))),
motrixsim_docs_root=Path(str(values.get("motrixsim_docs_root", os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_ROOT", "/opt/motrixsim-docs")))),
allowed_path_roots=_parse_path_tuple(str(values.get("allowed_path_roots", ""))),
worker_mode=str(values.get("worker_mode", os.environ.get("REAL2SIM_WORKER_MODE", "embedded"))),
worker_id=str(values.get("worker_id", os.environ.get("REAL2SIM_WORKER_ID", "local-worker"))),
@ -179,6 +182,7 @@ def write_default_config(path: Path, config: PlatformConfig | None = None) -> No
f'anysplat_python = "{cfg.anysplat_python.as_posix()}"',
f'cuda_home = "{cfg.cuda_home.as_posix()}"',
f'runtime_env_script = "{cfg.runtime_env_script.as_posix()}"',
f'motrixsim_docs_root = "{cfg.motrixsim_docs_root.as_posix()}"',
f'allowed_path_roots = "{_format_path_tuple(cfg.effective_allowed_path_roots)}"',
f'worker_mode = "{cfg.worker_mode}"',
f'worker_id = "{cfg.worker_id}"',

View File

@ -6,6 +6,7 @@ import math
import os
import shutil
import time
from copy import deepcopy
import xml.etree.ElementTree as ET
import zipfile
from pathlib import Path, PurePosixPath
@ -17,6 +18,66 @@ SCENE_MANIFEST_SCHEMA = "real2sim.scene_manifest.v3"
MOTRIX_IMPORT_SCHEMA = "real2sim.motrixsim_scene.v2"
SUPPORTED_MESH_SUFFIXES = {".obj", ".stl", ".ply", ".glb", ".gltf", ".usd", ".usda", ".usdc", ".usdz"}
SUPPORTED_PRIMITIVES = {"box", "sphere", "cylinder", "capsule"}
SCENE_COMPOSITION_SCHEMA = "real2sim.scene_composition.v1"
COMPOSITION_ASSET_RECIPES: dict[str, dict[str, Any]] = {
"recipe:coffee_table": {
"id": "recipe:coffee_table",
"name": "茶几",
"description": "静态碰撞茶几,适用于桌面操作任务",
"runtime_name": "coffee_table",
"category": "task_fixture",
"default_transform": {"translation": [0.65, 0.0, 0.25], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"physics": {"dynamic": False, "mass": 8.0, "collision_mode": "visual_and_collision", "dimensions": [0.9, 0.64, 0.34], "friction": [0.8, 0.01, 0.001], "rgba": [0.45, 0.30, 0.18, 1.0]},
"components": [
{"name": "top", "primitive": "box", "position": [0.0, 0.0, 0.0], "dimensions": [0.9, 0.64, 0.06], "rgba": [0.45, 0.30, 0.18, 1.0]},
{"name": "leg_1", "primitive": "box", "position": [0.36, 0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]},
{"name": "leg_2", "primitive": "box", "position": [-0.36, 0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]},
{"name": "leg_3", "primitive": "box", "position": [0.36, -0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]},
{"name": "leg_4", "primitive": "box", "position": [-0.36, -0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]},
],
"required_for_collection": True,
},
"recipe:pick_cube": {
"id": "recipe:pick_cube",
"name": "抓取方块",
"description": "带自由关节和真实接触的动态抓取目标",
"runtime_name": "pick_cube",
"category": "task_object",
"default_transform": {"translation": [0.65, 0.0, 0.33], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"physics": {"dynamic": True, "mass": 0.05, "collision_mode": "visual_and_collision", "dimensions": [0.06, 0.06, 0.06], "friction": [0.9, 0.01, 0.001], "rgba": [0.9, 0.2, 0.12, 1.0]},
"components": [{"name": "geom", "primitive": "box", "position": [0.0, 0.0, 0.0], "dimensions": [0.06, 0.06, 0.06], "rgba": [0.9, 0.2, 0.12, 1.0]}],
"required_for_collection": True,
},
"recipe:place_target": {
"id": "recipe:place_target",
"name": "放置目标",
"description": "不参与碰撞的半透明任务目标区域",
"runtime_name": "place_target",
"category": "task_marker",
"default_transform": {"translation": [0.45, 0.18, 0.335], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"physics": {"dynamic": False, "mass": 0.01, "collision_mode": "visual_only", "dimensions": [0.11, 0.11, 0.008], "friction": [0.0, 0.0, 0.0], "rgba": [0.1, 0.8, 0.6, 0.35]},
"components": [{"name": "marker", "primitive": "cylinder", "position": [0.0, 0.0, 0.0], "dimensions": [0.11, 0.11, 0.008], "rgba": [0.1, 0.8, 0.6, 0.35], "collision_mode": "visual_only"}],
"required_for_collection": True,
},
"recipe:storage_bin": {
"id": "recipe:storage_bin",
"name": "收纳盒",
"description": "由底板和四面碰撞壁组成的静态容器",
"runtime_name": "storage_bin",
"category": "prop",
"default_transform": {"translation": [0.45, -0.22, 0.37], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"physics": {"dynamic": False, "mass": 0.6, "collision_mode": "visual_and_collision", "dimensions": [0.28, 0.20, 0.12], "friction": [0.75, 0.01, 0.001], "rgba": [0.72, 0.72, 0.64, 1.0]},
"components": [
{"name": "base", "primitive": "box", "position": [0.0, 0.0, -0.05], "dimensions": [0.28, 0.20, 0.02]},
{"name": "wall_front", "primitive": "box", "position": [0.0, -0.095, 0.0], "dimensions": [0.28, 0.01, 0.12]},
{"name": "wall_back", "primitive": "box", "position": [0.0, 0.095, 0.0], "dimensions": [0.28, 0.01, 0.12]},
{"name": "wall_left", "primitive": "box", "position": [-0.135, 0.0, 0.0], "dimensions": [0.01, 0.18, 0.12]},
{"name": "wall_right", "primitive": "box", "position": [0.135, 0.0, 0.0], "dimensions": [0.01, 0.18, 0.12]},
],
"required_for_collection": False,
},
}
def detect_spatial_bundle(root: Path) -> dict[str, Any]:
@ -243,13 +304,18 @@ def compose_scene_asset(
if worldbody is None:
worldbody = ET.SubElement(root, "worldbody")
compiler = root.find("compiler")
compiler_angle = str(compiler.attrib.get("angle", "degree") if compiler is not None else "degree").lower()
euler = transform["rotation_euler_degrees"]
if compiler_angle.startswith("rad"):
euler = [math.radians(value) for value in euler]
body = ET.SubElement(
worldbody,
"body",
{
"name": asset_key,
"pos": _numbers(transform["translation"]),
"euler": _numbers(transform["rotation_euler_degrees"]),
"euler": _numbers(euler),
},
)
if physics["dynamic"]:
@ -348,6 +414,556 @@ def compose_scene_asset(
}
def composition_asset_catalog() -> list[dict[str, Any]]:
return [deepcopy(COMPOSITION_ASSET_RECIPES[key]) for key in sorted(COMPOSITION_ASSET_RECIPES)]
def build_scene_composition(
package_dir: Path,
*,
robot_source: Path,
robot_spec: dict[str, Any],
asset_specs: list[dict[str, Any]],
source_paths: dict[str, Path] | None,
validation_steps: int,
emit: Callable[[str], None] | None = None,
) -> dict[str, Any]:
emit = emit or (lambda _message: None)
source_paths = source_paths or {}
manifest_path = package_dir / "scene_manifest.json"
manifest = _read_json(manifest_path)
runtime_scene = _runtime_scene_from_manifest(package_dir, manifest)
if not runtime_scene.is_file():
raise FileNotFoundError(f"MotrixSim runtime scene not found: {runtime_scene}")
if not robot_source.is_file():
raise FileNotFoundError(f"RM65 source MJCF not found: {robot_source}")
tree = ET.parse(runtime_scene)
root = tree.getroot()
if root.tag != "mujoco":
raise ValueError("scene runtime file is not an MJCF <mujoco> document")
compiler = _ensure_radian_compiler(root)
emit("Merging the real RM65 articulated model into the MotrixSim scene")
robot_entry = _merge_rm65_robot(
root,
runtime_scene=runtime_scene,
compiler=compiler,
robot_source=robot_source,
robot_spec=robot_spec,
)
entries: list[dict[str, Any]] = []
seen_instances: set[str] = set()
seen_runtime_names: set[str] = set()
for index, spec in enumerate(asset_specs):
instance_id = _safe_name(str(spec.get("instance_id") or f"asset_{index + 1}"))
if instance_id in seen_instances:
raise ValueError(f"duplicate composition asset instance_id: {instance_id}")
seen_instances.add(instance_id)
catalog_id = str(spec.get("catalog_id") or "").strip()
if catalog_id in COMPOSITION_ASSET_RECIPES or catalog_id.startswith("primitive:"):
entry = _append_composition_primitive(root, spec=spec, instance_id=instance_id, seen_runtime_names=seen_runtime_names)
else:
source = source_paths.get(instance_id)
if source is None:
raise ValueError(f"composition asset is not registered or has no source file: {catalog_id or instance_id}")
entry = _append_composition_mesh(
root,
package_dir=package_dir,
runtime_scene=runtime_scene,
compiler=compiler,
spec=spec,
instance_id=instance_id,
source_path=source,
seen_runtime_names=seen_runtime_names,
emit=emit,
)
entries.append(entry)
runtime_names = {str(entry.get("runtime", {}).get("body") or "") for entry in entries}
collection_ready = {"coffee_table", "pick_cube", "place_target"}.issubset(runtime_names)
_ensure_overview_camera(root, target="pick_cube" if "pick_cube" in runtime_names else "base_link")
_indent_xml(root)
tree.write(runtime_scene, encoding="utf-8", xml_declaration=True)
_write_home_keyframe(runtime_scene)
robot_glb = package_dir / "viewer" / "robots" / "rm65.glb"
emit("Exporting the articulated RM65 visual geometry for the browser editor")
robot_preview = export_mujoco_body_glb(runtime_scene, robot_glb, root_body="base_link")
robot_entry.update(
{
"path": str(robot_glb),
"size_bytes": robot_glb.stat().st_size,
"sha256": _sha256(robot_glb),
"preview": robot_preview,
}
)
emit("Validating the composed scene in MuJoCo and MotrixSim")
validation = validate_runtime_scene(runtime_scene, steps=validation_steps)
reports_dir = package_dir / "reports"
reports_dir.mkdir(parents=True, exist_ok=True)
validation_path = reports_dir / "scene_composition_validation.json"
_write_json(validation_path, validation)
if not validation["ok"]:
failed = [name for name in ("mujoco", "motrixsim") if not validation[name]["ok"]]
raise RuntimeError("Composed scene failed runtime validation: " + "; ".join(f"{name}: {validation[name].get('error', 'unknown error')}" for name in failed))
for entry in [robot_entry, *entries]:
entry.setdefault("runtime", {})["validated"] = True
composition = {
"schema": SCENE_COMPOSITION_SCHEMA,
"simulator": "motrixsim",
"robot": robot_entry,
"assets": entries,
"controller": "rm65_pick_place_6d_ik" if collection_ready else None,
"collection_ready": collection_ready,
"required_collection_objects": ["coffee_table", "pick_cube", "place_target"],
"validation_steps": validation_steps,
"updated_at": _utc_now(),
}
report_path = reports_dir / "scene_composition_report.json"
_write_json(
report_path,
{
"schema": "real2sim.scene_composition_report.v1",
"ok": True,
"runtime_scene": str(runtime_scene),
"robot_source": str(robot_source),
"robot_body_count": robot_preview["body_count"],
"robot_mesh_count": robot_preview["mesh_count"],
"asset_count": len(entries),
"collection_ready": collection_ready,
"runtime_validation": validation,
},
)
existing_assets = [
item
for item in manifest.get("assets", [])
if not (isinstance(item, dict) and item.get("role") in {"robot", "added_asset"})
]
manifest["assets"] = [robot_entry, *entries, *existing_assets]
manifest["composition"] = composition
manifest["updated_at"] = _utc_now()
manifest.setdefault("asset_editing", {}).update({"enabled": True, "revisioned": True, "composition": True})
manifest.setdefault("simulator_artifacts", {}).setdefault("motrixsim", {}).update({"runtime_scene": str(runtime_scene), "validated": True})
manifest.setdefault("simulator_artifacts", {}).setdefault("mujoco", {}).update({"mjcf": str(runtime_scene), "validated": True})
_write_json(manifest_path, manifest)
descriptor_path = package_dir / "motrix_import.json"
descriptor = _read_json(descriptor_path) if descriptor_path.is_file() else {"schema": MOTRIX_IMPORT_SCHEMA}
descriptor.update({"runtime_scene": str(runtime_scene), "composition": composition, "runtime_validation": validation, "updated_at": _utc_now()})
_write_json(descriptor_path, descriptor)
return {
"manifest_path": str(manifest_path),
"runtime_scene": str(runtime_scene),
"validation_path": str(validation_path),
"report_path": str(report_path),
"validation": validation,
"composition": composition,
"collection_ready": collection_ready,
}
def export_mujoco_body_glb(scene_path: Path, output: Path, *, root_body: str, relative_to_root: bool = False) -> dict[str, Any]:
try:
import mujoco
import numpy as np
import trimesh
except ImportError as exc:
raise RuntimeError("RM65 GLB export requires mujoco, numpy and trimesh") from exc
model = mujoco.MjModel.from_xml_path(str(scene_path))
data = mujoco.MjData(model)
if model.nkey:
mujoco.mj_resetDataKeyframe(model, data, 0)
mujoco.mj_forward(model, data)
root_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, root_body)
if root_id < 0:
raise ValueError(f"robot root body is missing: {root_body}")
root_position = np.asarray(data.xpos[root_id], dtype=np.float64)
root_rotation = np.asarray(data.xmat[root_id], dtype=np.float64).reshape(3, 3)
def is_descendant(body_id: int) -> bool:
current = body_id
while current > 0:
if current == root_id:
return True
current = int(model.body_parentid[current])
return current == root_id
scene = trimesh.Scene()
mesh_count = 0
body_ids: set[int] = set()
for geom_id in range(model.ngeom):
body_id = int(model.geom_bodyid[geom_id])
mesh_id = int(model.geom_dataid[geom_id])
if not is_descendant(body_id) or int(model.geom_type[geom_id]) != int(mujoco.mjtGeom.mjGEOM_MESH) or mesh_id < 0:
continue
vertex_start = int(model.mesh_vertadr[mesh_id])
vertex_count = int(model.mesh_vertnum[mesh_id])
face_start = int(model.mesh_faceadr[mesh_id])
face_count = int(model.mesh_facenum[mesh_id])
vertices = np.asarray(model.mesh_vert[vertex_start : vertex_start + vertex_count], dtype=np.float64).copy()
faces = np.asarray(model.mesh_face[face_start : face_start + face_count], dtype=np.int64).copy()
rotation = np.asarray(data.geom_xmat[geom_id], dtype=np.float64).reshape(3, 3)
vertices = vertices @ rotation.T + np.asarray(data.geom_xpos[geom_id], dtype=np.float64)
if relative_to_root:
vertices = (vertices - root_position) @ root_rotation
mesh = trimesh.Trimesh(vertices=vertices, faces=faces, process=False, validate=False)
rgba = np.clip(np.asarray(model.geom_rgba[geom_id]) * 255, 0, 255).astype(np.uint8)
mesh.visual.vertex_colors = np.tile(rgba, (len(mesh.vertices), 1))
name = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, geom_id) or f"rm65_geom_{geom_id}"
scene.add_geometry(mesh, geom_name=name, node_name=name)
mesh_count += 1
body_ids.add(body_id)
if not mesh_count:
raise ValueError("RM65 model contains no exportable visual mesh geometry")
output.parent.mkdir(parents=True, exist_ok=True)
scene.export(output, file_type="glb")
return {"path": str(output), "mesh_count": mesh_count, "body_count": len(body_ids), "size_bytes": output.stat().st_size}
def _ensure_radian_compiler(root: ET.Element) -> ET.Element:
compiler = root.find("compiler")
if compiler is None:
compiler = ET.Element("compiler", {"angle": "radian", "autolimits": "true"})
root.insert(0, compiler)
return compiler
if not str(compiler.attrib.get("angle", "degree")).lower().startswith("rad"):
for element in root.iter():
if "euler" not in element.attrib:
continue
values = [float(value) for value in element.attrib["euler"].split()]
element.attrib["euler"] = _numbers([math.radians(value) for value in values])
compiler.attrib["angle"] = "radian"
compiler.attrib.setdefault("autolimits", "true")
return compiler
def _merge_rm65_robot(
root: ET.Element,
*,
runtime_scene: Path,
compiler: ET.Element,
robot_source: Path,
robot_spec: dict[str, Any],
) -> dict[str, Any]:
if str(robot_spec.get("robot_id") or "") != "rm65":
raise ValueError("only the registered RM65 robot can be composed in this release")
if root.find("./worldbody/body[@name='base_link']") is not None:
raise ValueError("the scene already contains an RM65 base_link body")
transform = normalize_transform(robot_spec.get("transform") or {})
if any(abs(value - 1.0) > 1e-9 for value in transform["scale"]):
raise ValueError("articulated RM65 scale must remain [1, 1, 1] to preserve joint and inertia calibration")
robot_root = ET.parse(robot_source).getroot()
robot_compiler = robot_root.find("compiler")
asset_dir = (robot_source.parent / str(robot_compiler.attrib.get("assetdir", "assets") if robot_compiler is not None else "assets")).resolve()
if not asset_dir.is_dir():
raise FileNotFoundError(f"RM65 asset directory is missing: {asset_dir}")
mesh_dir = (runtime_scene.parent / str(compiler.attrib.get("meshdir") or ".")).resolve()
robot_mesh_dir = mesh_dir / "robots" / "rm65"
if robot_mesh_dir.exists():
shutil.rmtree(robot_mesh_dir)
shutil.copytree(asset_dir, robot_mesh_dir)
license_source = robot_source.parent / "LICENSE"
if license_source.is_file():
shutil.copy2(license_source, robot_mesh_dir / "LICENSE")
robot_default = robot_root.find("default")
if robot_default is not None:
target_default = root.find("default")
if target_default is None:
target_default = ET.Element("default")
_insert_before(root, target_default, {"asset", "worldbody"})
for child in robot_default:
target_default.append(deepcopy(child))
target_asset = root.find("asset")
if target_asset is None:
target_asset = ET.Element("asset")
_insert_before(root, target_asset, {"worldbody"})
source_asset = robot_root.find("asset")
if source_asset is not None:
for child in source_asset:
copied = deepcopy(child)
if copied.get("file"):
copied.set("file", f"robots/rm65/{copied.get('file')}")
target_asset.append(copied)
worldbody = root.find("worldbody")
if worldbody is None:
worldbody = ET.SubElement(root, "worldbody")
robot_body = robot_root.find("./worldbody/body[@name='base_link']")
if robot_body is None:
raise ValueError("RM65 source MJCF does not contain base_link")
copied_body = deepcopy(robot_body)
copied_body.set("pos", _numbers(transform["translation"]))
copied_body.set("euler", _numbers([math.radians(value) for value in transform["rotation_euler_degrees"]]))
worldbody.append(copied_body)
for section_name in ("equality", "actuator", "contact"):
source_section = robot_root.find(section_name)
if source_section is None:
continue
target_section = root.find(section_name)
if target_section is None:
target_section = ET.SubElement(root, section_name)
for child in source_section:
target_section.append(deepcopy(child))
return {
"name": "RM65 双指夹爪",
"kind": "robot",
"role": "robot",
"format": "glb",
"autoload": True,
"viewer_hint": "overlay",
"transform": {"translation": [0.0, 0.0, 0.0], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"source_transform": transform,
"robot_id": "rm65",
"controller": "rm65_pick_place_6d_ik",
"runtime": {"body": "base_link", "scene": str(runtime_scene), "validated": False},
"source": {"path": str(robot_source), "repository": "Motphys/motrixsim-docs"},
"added_at": _utc_now(),
}
def _append_composition_primitive(
root: ET.Element,
*,
spec: dict[str, Any],
instance_id: str,
seen_runtime_names: set[str],
) -> dict[str, Any]:
catalog_id = str(spec.get("catalog_id") or "")
if catalog_id.startswith("primitive:"):
primitive = catalog_id.split(":", 1)[1]
if primitive not in SUPPORTED_PRIMITIVES:
raise ValueError(f"unsupported composition primitive: {primitive}")
recipe = {
"id": catalog_id,
"name": str(spec.get("name") or primitive),
"runtime_name": instance_id,
"default_transform": {"translation": [0.0, 0.0, 0.1], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"physics": {"dynamic": False, "mass": 0.25, "collision_mode": "visual_and_collision", "dimensions": [0.1, 0.1, 0.1], "friction": [0.8, 0.01, 0.001], "rgba": [0.13, 0.76, 0.67, 1.0]},
"components": [{"name": "geom", "primitive": primitive, "position": [0.0, 0.0, 0.0], "dimensions": [0.1, 0.1, 0.1]}],
}
else:
recipe = COMPOSITION_ASSET_RECIPES[catalog_id]
runtime_name = str(recipe["runtime_name"])
if runtime_name in seen_runtime_names or root.find(f"./worldbody/body[@name='{runtime_name}']") is not None:
raise ValueError(f"composition already contains runtime object: {runtime_name}")
seen_runtime_names.add(runtime_name)
transform = normalize_transform(spec.get("transform") or recipe.get("default_transform") or {})
physics = normalize_physics({**(recipe.get("physics") or {}), **(spec.get("physics") or {})})
worldbody = root.find("worldbody")
if worldbody is None:
worldbody = ET.SubElement(root, "worldbody")
body = ET.SubElement(
worldbody,
"body",
{
"name": runtime_name,
"pos": _numbers(transform["translation"]),
"euler": _numbers([math.radians(value) for value in transform["rotation_euler_degrees"]]),
},
)
if physics["dynamic"]:
joint_name = "pick_cube_free" if runtime_name == "pick_cube" else f"{runtime_name}_free"
ET.SubElement(body, "freejoint", {"name": joint_name})
viewer_components: list[dict[str, Any]] = []
collidable = [component for component in recipe["components"] if component.get("collision_mode", physics["collision_mode"]) != "visual_only"]
mass_per_geom = physics["mass"] / max(1, len(collidable))
for component in recipe["components"]:
primitive = str(component["primitive"])
dimensions = [float(value) for value in component.get("dimensions") or physics["dimensions"]]
scaled_dimensions = [value * transform["scale"][index] for index, value in enumerate(dimensions)]
local_position = [float(value) for value in component.get("position") or [0.0, 0.0, 0.0]]
scaled_position = [value * transform["scale"][index] for index, value in enumerate(local_position)]
collision_mode = str(component.get("collision_mode") or physics["collision_mode"])
rgba = [float(value) for value in component.get("rgba") or physics["rgba"]]
component_name = str(component.get("name") or "geom")
if runtime_name == "pick_cube":
geom_name = "pick_cube_geom"
elif runtime_name == "place_target":
geom_name = "place_target_marker"
elif runtime_name == "coffee_table" and component_name == "top":
geom_name = "coffee_table_top"
else:
geom_name = f"{runtime_name}_{_safe_name(component_name)}"
attributes = {
"name": geom_name,
"pos": _numbers(scaled_position),
"friction": _numbers(physics["friction"]),
"contype": "0" if collision_mode == "visual_only" else "1",
"conaffinity": "0" if collision_mode == "visual_only" else "1",
"group": "2" if collision_mode == "visual_only" else "3",
"rgba": _numbers(rgba),
**_primitive_geom(primitive, scaled_dimensions),
}
if physics["dynamic"] and collision_mode != "visual_only":
attributes["mass"] = f"{mass_per_geom:.9g}"
ET.SubElement(body, "geom", attributes)
viewer_components.append(
{
"name": component_name,
"viewer_primitive": primitive,
"translation": local_position,
"rotation_euler_degrees": component.get("rotation_euler_degrees") or [0.0, 0.0, 0.0],
"dimensions": dimensions,
"rgba": rgba,
}
)
return {
"name": str(spec.get("name") or recipe["name"]),
"path": "",
"kind": "primitive" if len(viewer_components) == 1 else "compound_primitive",
"role": "added_asset",
"format": str(viewer_components[0]["viewer_primitive"]) if len(viewer_components) == 1 else "compound",
"autoload": True,
"viewer_hint": "overlay",
"viewer_primitive": viewer_components[0]["viewer_primitive"] if len(viewer_components) == 1 else None,
"viewer_components": viewer_components,
"catalog_id": catalog_id,
"instance_id": instance_id,
"transform": transform,
"physics": physics,
"runtime": {"body": runtime_name, "validated": False},
"added_at": _utc_now(),
}
def _append_composition_mesh(
root: ET.Element,
*,
package_dir: Path,
runtime_scene: Path,
compiler: ET.Element,
spec: dict[str, Any],
instance_id: str,
source_path: Path,
seen_runtime_names: set[str],
emit: Callable[[str], None],
) -> dict[str, Any]:
runtime_name = _safe_name(f"asset_{instance_id}")
if runtime_name in seen_runtime_names or root.find(f"./worldbody/body[@name='{runtime_name}']") is not None:
raise ValueError(f"composition already contains runtime object: {runtime_name}")
seen_runtime_names.add(runtime_name)
transform = normalize_transform(spec.get("transform") or {})
physics = normalize_physics(spec.get("physics") or {})
added_dir = package_dir / "assets" / "added" / instance_id
added_dir.mkdir(parents=True, exist_ok=True)
copied_asset = added_dir / source_path.name
shutil.copy2(source_path, copied_asset)
runtime_mesh = added_dir / f"{runtime_name}.obj"
emit(f"Converting {source_path.name} into a MotrixSim composition mesh")
convert_asset_to_obj(copied_asset, runtime_mesh)
viewer_asset = copied_asset if copied_asset.suffix.lower() in {".glb", ".gltf", ".obj"} else runtime_mesh
mesh_dir = (runtime_scene.parent / str(compiler.attrib.get("meshdir") or ".")).resolve()
relative_mesh = os.path.relpath(runtime_mesh, mesh_dir).replace(os.sep, "/")
asset_node = root.find("asset")
if asset_node is None:
asset_node = ET.Element("asset")
_insert_before(root, asset_node, {"worldbody"})
mesh_name = f"{runtime_name}_mesh"
ET.SubElement(asset_node, "mesh", {"name": mesh_name, "file": relative_mesh, "scale": _numbers(transform["scale"])})
worldbody = root.find("worldbody")
if worldbody is None:
worldbody = ET.SubElement(root, "worldbody")
body = ET.SubElement(
worldbody,
"body",
{"name": runtime_name, "pos": _numbers(transform["translation"]), "euler": _numbers([math.radians(value) for value in transform["rotation_euler_degrees"]])},
)
if physics["dynamic"]:
ET.SubElement(body, "freejoint", {"name": f"{runtime_name}_free"})
attributes = {
"name": f"{runtime_name}_geom",
"type": "mesh",
"mesh": mesh_name,
"friction": _numbers(physics["friction"]),
"contype": "0" if physics["collision_mode"] == "visual_only" else "1",
"conaffinity": "0" if physics["collision_mode"] == "visual_only" else "1",
"group": "2" if physics["collision_mode"] == "visual_only" else "3",
"rgba": _numbers(physics["rgba"]),
}
if physics["dynamic"]:
attributes["mass"] = f"{physics['mass']:.9g}"
ET.SubElement(body, "geom", attributes)
return {
"name": str(spec.get("name") or source_path.stem),
"path": str(viewer_asset),
"kind": "mesh",
"role": "added_asset",
"format": viewer_asset.suffix.lower().lstrip("."),
"size_bytes": viewer_asset.stat().st_size,
"autoload": True,
"viewer_hint": "overlay",
"catalog_id": str(spec.get("catalog_id") or ""),
"instance_id": instance_id,
"transform": transform,
"physics": physics,
"runtime": {"body": runtime_name, "validated": False},
"added_at": _utc_now(),
}
def _ensure_overview_camera(root: ET.Element, *, target: str) -> None:
worldbody = root.find("worldbody")
if worldbody is None:
worldbody = ET.SubElement(root, "worldbody")
existing = worldbody.find("./camera[@name='overview']")
if existing is None:
ET.SubElement(worldbody, "camera", {"name": "overview", "mode": "targetbody", "target": target, "pos": "2 -2 1.8", "fovy": "55"})
def _write_home_keyframe(runtime_scene: Path) -> None:
try:
import mujoco
import numpy as np
except ImportError as exc:
raise RuntimeError("composition keyframe generation requires mujoco and numpy") from exc
tree = ET.parse(runtime_scene)
root = tree.getroot()
keyframe = root.find("keyframe")
if keyframe is None:
keyframe = ET.SubElement(root, "keyframe")
for child in list(keyframe):
if child.tag == "key" and child.get("name") == "home":
keyframe.remove(child)
home = ET.Element("key", {"name": "home"})
keyframe.insert(0, home)
_indent_xml(root)
tree.write(runtime_scene, encoding="utf-8", xml_declaration=True)
model = mujoco.MjModel.from_xml_path(str(runtime_scene))
qpos = np.asarray(model.qpos0, dtype=float).copy()
arm_home = [0.0, 0.0, 0.0, -1.57079, 0.0, 1.57079]
for index, value in enumerate(arm_home, start=1):
joint_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}")
if joint_id >= 0:
qpos[int(model.jnt_qposadr[joint_id])] = value
gripper_joint = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint")
if gripper_joint >= 0:
qpos[int(model.jnt_qposadr[gripper_joint])] = -0.7853
controls = np.zeros(model.nu, dtype=float)
controls[: min(6, model.nu)] = arm_home[: min(6, model.nu)]
if model.nu >= 7:
controls[6] = -0.7853
home.set("qpos", _numbers(qpos.tolist()))
if model.nu:
home.set("ctrl", _numbers(controls.tolist()))
_indent_xml(root)
tree.write(runtime_scene, encoding="utf-8", xml_declaration=True)
mujoco.MjModel.from_xml_path(str(runtime_scene))
def _insert_before(root: ET.Element, child: ET.Element, before_tags: set[str]) -> None:
index = next((position for position, existing in enumerate(list(root)) if existing.tag in before_tags), len(root))
root.insert(index, child)
def load_usd_meshes(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]:
try:
import numpy as np

View File

@ -51,6 +51,7 @@ class RM65PickPlaceRuntime:
self._id(mujoco.mjtObj.mjOBJ_GEOM, "right_finger_pad_z_axis"),
}
self.target_body = self._id(mujoco.mjtObj.mjOBJ_BODY, "place_target")
self.target_home = self.model.body_pos[self.target_body].astype(float).copy()
self.gripper_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint")
self.gripper_qadr = int(self.model.jnt_qposadr[self.gripper_joint])
self.joint_ids = [self._id(mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}") for index in range(1, 7)]
@ -70,12 +71,20 @@ class RM65PickPlaceRuntime:
def run_episode(self, *, max_steps: int) -> EpisodeResult:
mujoco = self.mujoco
mujoco.mj_resetDataKeyframe(self.model, self.data, 0)
cube = np.array([self.rng.uniform(0.645, 0.655), self.rng.uniform(-0.010, 0.010), 0.33], dtype=float)
target = np.array([self.rng.uniform(0.42, 0.51), self.rng.uniform(0.13, 0.22), 0.31], dtype=float)
cube_home = self.data.qpos[self.cube_qadr : self.cube_qadr + 3].astype(float).copy()
cube = cube_home + np.array([self.rng.uniform(-0.010, 0.010), self.rng.uniform(-0.010, 0.010), 0.0], dtype=float)
target = np.array(
[
self.target_home[0] + self.rng.uniform(-0.025, 0.025),
self.target_home[1] + self.rng.uniform(-0.025, 0.025),
self.target_home[2] - 0.025,
],
dtype=float,
)
self.data.qpos[self.cube_qadr : self.cube_qadr + 3] = cube
self.data.qpos[self.cube_qadr + 3 : self.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
self.data.qvel[self.model.jnt_dofadr[self.cube_joint] : self.model.jnt_dofadr[self.cube_joint] + 6] = 0
self.model.body_pos[self.target_body] = np.array([target[0], target[1], 0.335])
self.model.body_pos[self.target_body] = np.array([target[0], target[1], self.target_home[2]])
mujoco.mj_forward(self.model, self.data)
for _ in range(120):
mujoco.mj_step(self.model, self.data)

View File

@ -27,7 +27,7 @@ from . import __version__
from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset
from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username
from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config
from .motrix_scene import SUPPORTED_MESH_SUFFIXES, SUPPORTED_PRIMITIVES
from .motrix_scene import COMPOSITION_ASSET_RECIPES, SUPPORTED_MESH_SUFFIXES, SUPPORTED_PRIMITIVES, composition_asset_catalog
from .source_revision import read_source_revision
from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file
from .store import PlatformStore
@ -58,6 +58,28 @@ class SceneAssetRequest(BaseModel):
revision_name: str | None = None
class SceneCompositionRobotRequest(BaseModel):
robot_id: str
transform: dict[str, Any] = Field(default_factory=dict)
class SceneCompositionAssetRequest(BaseModel):
catalog_id: str
instance_id: str
name: str | None = None
source: dict[str, Any] = Field(default_factory=dict)
path: str = ""
transform: dict[str, Any] = Field(default_factory=dict)
physics: dict[str, Any] = Field(default_factory=dict)
class SceneCompositionRequest(BaseModel):
name: str | None = None
robot: SceneCompositionRobotRequest
assets: list[SceneCompositionAssetRequest] = Field(default_factory=list)
validation_steps: int = 120
class CreateTaskRequest(BaseModel):
type: str
params: dict[str, Any] = {}
@ -307,6 +329,7 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
"motrixsim": True,
"motrixsim_scene_import": True,
"scene_asset_compose": True,
"scene_composition": True,
},
}
@ -594,6 +617,10 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
def list_digital_assets() -> list[dict[str, Any]]:
return _digital_asset_catalog(store)
@app.get("/api/robots")
def list_robots() -> list[dict[str, Any]]:
return _robot_catalog(cfg)
@app.post("/api/scenes/import")
def import_scene(payload: ImportSceneRequest) -> dict[str, Any]:
path = Path(payload.path)
@ -650,6 +677,27 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
"message": "数字资产合成任务已创建;通过 MotrixSim 与 MuJoCo 校验后会生成新场景版本",
}
@app.post("/api/scenes/{scene_id}/compose")
def compose_scene(scene_id: str, payload: SceneCompositionRequest) -> dict[str, Any]:
try:
store.get_scene(scene_id)
except KeyError as exc:
raise HTTPException(status_code=404, detail="scene not found") from exc
composition = payload.model_dump(exclude_none=True)
params = {
"scene_id": scene_id,
"name": composition.pop("name", None),
"validation_steps": composition.pop("validation_steps", 120),
"composition": composition,
}
task = _queue_task("scene_composition", params)
return {
"ok": True,
"scene_id": scene_id,
"task": _with_task_view_model(task),
"message": "场景组合任务已创建;RM65、数字资产与 MotrixSim 运行时校验完成后会生成独立场景版本",
}
@app.patch("/api/scenes/{scene_id}")
def update_scene(scene_id: str, payload: UpdateEntityRequest) -> dict[str, Any]:
try:
@ -696,7 +744,7 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
payload = CreateTaskRequest(type=task_type, params=params)
_validate_task_payload(payload, cfg)
_validate_task_resources(store, payload)
if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation"}:
if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "scene_composition", "simulator_validation"}:
raise ValueError(f"unknown task type: {payload.type}")
run_dir = cfg.runs_dir / payload.type / _next_run_name(cfg.runs_dir / payload.type, payload.type)
return store.create_task(task_type=payload.type, params=payload.params, run_dir=run_dir)
@ -1169,13 +1217,13 @@ def _metrics_summary(kind: str, metrics: dict[str, Any], scan: dict[str, Any], m
def _resource_subtitle(kind: str, entity: dict[str, Any], manifest: dict[str, Any]) -> str:
source = str(entity.get("source_type") or manifest.get("source_type") or "")
version = str(entity.get("version") or "")
label = {"prepared_media": "已准备帧数据", "expert_collection": "专家采集数据", "scene_reconstruction": "场景重建", "motrixsim_spatial_bundle": "MotrixSim 空间包", "scene_revision": "场景修订", "huggingface": "HuggingFace", "upload": "上传文件", "browser_upload": "浏览器上传", "local_path": "本地导入"}.get(source, source or kind)
label = {"prepared_media": "已准备帧数据", "expert_collection": "专家采集数据", "scene_reconstruction": "场景重建", "motrixsim_spatial_bundle": "MotrixSim 空间包", "scene_revision": "场景修订", "scene_composition": "场景组合", "huggingface": "HuggingFace", "upload": "上传文件", "browser_upload": "浏览器上传", "local_path": "本地导入"}.get(source, source or kind)
return f"{label}{' · ' + version if version else ''}"
def _task_display_name(task: dict[str, Any]) -> str:
params = task.get("params") or {}
return str(params.get("scene_name") or params.get("output_name") or _task_type_label(str(task.get("type") or "")))
return str(params.get("scene_name") or params.get("output_name") or params.get("name") or _task_type_label(str(task.get("type") or "")))
def _task_type_label(task_type: str) -> str:
@ -1187,6 +1235,7 @@ def _task_type_label(task_type: str) -> str:
"motrixsim_asset_import": "MotrixSim 资产导入",
"motrixsim_scene_import": "MotrixSim 场景包导入",
"scene_asset_compose": "数字资产合成",
"scene_composition": "场景组合",
"simulator_validation": "仿真校验",
"expert_collection": "专家采集",
"training": "模型训练",
@ -1234,6 +1283,30 @@ def _validate_task_payload(payload: CreateTaskRequest, cfg: PlatformConfig) -> N
raise HTTPException(status_code=400, detail="asset file not found")
if not is_path_allowed(path, cfg):
raise HTTPException(status_code=403, detail="asset path is outside configured import roots")
if payload.type == "scene_composition":
if not params.get("scene_id"):
raise HTTPException(status_code=400, detail="scene_composition requires scene_id")
composition = params.get("composition") or {}
robot = composition.get("robot") or {}
if robot.get("robot_id") != "rm65":
raise HTTPException(status_code=400, detail="scene_composition currently requires the registered RM65 robot")
assets = composition.get("assets") or []
if not isinstance(assets, list) or len(assets) > 50:
raise HTTPException(status_code=400, detail="scene_composition supports at most 50 asset instances")
instance_ids = [str(item.get("instance_id") or "") for item in assets if isinstance(item, dict)]
if any(not value for value in instance_ids) or len(instance_ids) != len(set(instance_ids)):
raise HTTPException(status_code=400, detail="composition asset instance_id values must be present and unique")
for item in assets:
catalog_id = str(item.get("catalog_id") or "")
source = item.get("source") or {}
if catalog_id in COMPOSITION_ASSET_RECIPES:
continue
if catalog_id.startswith("primitive:") and catalog_id.split(":", 1)[1] in SUPPORTED_PRIMITIVES:
continue
if source.get("type") != "dataset" or not (source.get("dataset_id") and source.get("relative_path")):
raise HTTPException(status_code=400, detail=f"unregistered composition asset requires a dataset source: {catalog_id}")
if not 1 <= int(params.get("validation_steps", 120)) <= 5000:
raise HTTPException(status_code=400, detail="validation_steps must be between 1 and 5000")
if payload.type == "hf_import":
repo_type = params.get("repo_type", "dataset")
if repo_type != "dataset":
@ -1321,6 +1394,20 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) -
raise HTTPException(status_code=400, detail="资产路径越出所选数据集")
if not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_MESH_SUFFIXES:
raise HTTPException(status_code=400, detail="所选数据集文件不是可导入 MotrixSim 的网格资产")
if payload.type == "scene_composition":
scene = store.get_scene(str(params["scene_id"]))
package = Path(str((scene.get("metadata") or {}).get("package") or ""))
if not package.is_dir() or not (package / "scene_manifest.json").is_file():
raise HTTPException(status_code=400, detail="所选场景不是可编辑的 MotrixSim 场景包")
for item in (params.get("composition") or {}).get("assets") or []:
source = item.get("source") or {}
if source.get("type") != "dataset":
continue
dataset = store.get_dataset(str(source.get("dataset_id") or ""))
root = Path(dataset["path"]).resolve()
candidate = (root / str(source.get("relative_path") or "")).resolve()
if candidate == root or root not in candidate.parents or not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_MESH_SUFFIXES:
raise HTTPException(status_code=400, detail="组合资产不是数据集内可导入 MotrixSim 的网格文件")
if payload.type == "training" and params.get("dataset_id"):
dataset = store.get_dataset(str(params["dataset_id"]))
source = str(dataset.get("source_type") or "")
@ -1342,6 +1429,15 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) -
def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]:
primitive_names = {"box": "立方体", "sphere": "球体", "cylinder": "圆柱体", "capsule": "胶囊体"}
items: list[dict[str, Any]] = [
{
**recipe,
"source_type": "recipe",
"format": "compound" if len(recipe.get("components") or []) > 1 else str((recipe.get("components") or [{}])[0].get("primitive") or "primitive"),
"source": {"type": "recipe", "recipe_id": recipe["id"]},
"capabilities": {"motrixsim_physics": True, "browser_preview": True, "dynamic": bool((recipe.get("physics") or {}).get("dynamic")), "collision": str((recipe.get("physics") or {}).get("collision_mode")) != "visual_only"},
}
for recipe in composition_asset_catalog()
] + [
{
"id": f"primitive:{primitive}",
"name": primitive_names.get(primitive, primitive),
@ -1379,6 +1475,7 @@ def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]:
"relative_path": relative.as_posix(),
"format": suffix.lstrip("."),
"size_bytes": candidate.stat().st_size,
"preview_url": _workspace_public_url(candidate) if suffix in {".obj", ".glb", ".gltf", ".ply"} else "",
"source": {"type": "dataset", "dataset_id": dataset["id"], "relative_path": relative.as_posix()},
"capabilities": {
"motrixsim_physics": True,
@ -1391,6 +1488,30 @@ def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]:
return items
def _robot_catalog(cfg: PlatformConfig) -> list[dict[str, Any]]:
return [
{
"id": "rm65",
"name": "RM65 双指夹爪",
"manufacturer": "睿尔曼",
"simulator": "motrixsim",
"description": "6 轴机械臂、双指夹爪、7 个位置执行器和真实碰撞体",
"preview_url": "/static/assets/robots/rm65.glb",
"controller": "rm65_pick_place_6d_ik",
"joint_count": 6,
"actuator_count": 7,
"payload_kg": 5,
"default_transform": {"translation": [1.03, 0.0, 0.011], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]},
"source": {
"repository": "Motphys/motrixsim-docs",
"commit": os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_COMMIT", "2beecea67c81d25e29cefa30a80c0ddad7b44dde"),
"mjcf": "examples/assets/rm65_open_cabinet/mjx_rm65.xml",
},
"capabilities": {"motrixsim": True, "mujoco": True, "browser_preview": True, "pick_place": True},
}
]
def _active_admin_count(store: PlatformStore) -> int:
return sum(1 for user in store.list_users() if user["role"] == "admin" and not user["disabled"])
@ -1512,6 +1633,7 @@ def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]:
"gsplat_python": str(cfg.gsplat_python),
"cuda_home": str(cfg.cuda_home),
"runtime_env_script": str(cfg.runtime_env_script),
"motrixsim_docs_root": str(cfg.motrixsim_docs_root),
"discoverse_commit": cfg.discoverse_commit,
"act_commit": cfg.act_commit,
"anysplat_commit": cfg.anysplat_commit,

View File

@ -1,4 +1,4 @@
const STATIC_VERSION = "20260712-motrix-spatial-v4";
const STATIC_VERSION = "20260712-scene-composition-v1";
const state = {
datasets: [],
@ -11,6 +11,7 @@ const state = {
editing: null,
assetLibrary: [],
digitalAssets: [],
robots: [],
refreshTimer: null,
taskEvents: null,
selections: {},
@ -19,6 +20,21 @@ const state = {
recipes: null,
initialized: false,
pendingPassword: "",
composition: {
baseSceneId: "",
robotId: "",
robotTransform: null,
assets: [],
selectedId: "",
tool: "select",
history: [],
future: [],
bounds: null,
previewSceneId: "",
savedSceneId: "",
collectionReady: false,
pendingTaskId: "",
},
};
const $ = (id) => document.getElementById(id);
@ -26,6 +42,7 @@ const $ = (id) => document.getElementById(id);
const taskScopes = {
dataTaskList: ["hf_import", "media_prepare"],
sceneTaskList: ["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"],
compositionTaskList: ["scene_composition", "simulator_validation"],
collectTaskList: ["expert_collection"],
trainTaskList: ["training"],
inferTaskList: ["inference"],
@ -90,6 +107,7 @@ function startTaskEvents() {
state.tasks = JSON.parse(event.data);
renderScopedTaskLists();
renderOverview();
renderComposition();
if (state.selectedTaskId && $("taskDrawer")?.classList.contains("open")) showTask(state.selectedTaskId, { preservePreview: true }).catch(console.warn);
if (state.tasks.some((task) => terminalStatus(task.status) && !previousTerminal.has(task.id))) refresh({ quiet: true }).catch(console.warn);
});
@ -125,18 +143,20 @@ function fillRecipeSelect(id, items, emptyLabel) {
async function refresh({ quiet = false } = {}) {
try {
const [datasets, scenes, models, tasks, digitalAssets] = await Promise.all([
const [datasets, scenes, models, tasks, digitalAssets, robots] = await Promise.all([
api("/api/datasets"),
api("/api/scenes"),
api("/api/models"),
api("/api/tasks"),
api("/api/assets"),
api("/api/robots"),
]);
state.datasets = Array.isArray(datasets) ? datasets : [];
state.scenes = Array.isArray(scenes) ? scenes : [];
state.models = Array.isArray(models) ? models : [];
state.tasks = Array.isArray(tasks) ? tasks : [];
state.digitalAssets = Array.isArray(digitalAssets) ? digitalAssets : [];
state.robots = Array.isArray(robots) ? robots : [];
if (state.activeSceneId && !state.scenes.some((scene) => scene.id === state.activeSceneId)) {
state.activeSceneId = "";
}
@ -159,6 +179,7 @@ function renderAll() {
renderDatasetList();
renderReconstructDatasetCards();
renderSceneList();
renderComposition();
renderPreflightSummary();
renderCollectionDatasets();
renderModelList();
@ -175,10 +196,11 @@ function updateActionAvailability() {
setActionAvailability("createMediaPrepare", writable && state.datasets.length > 0, writable ? "请先新增数据集" : "只读用户不能创建任务");
setActionAvailability("createPreflightTask", writable && preparedDatasets().length > 0, "请先准备帧数据");
setActionAvailability("createSceneTask", writable && preparedDatasets().length > 0, "请先准备帧数据");
setActionAvailability("createCollectTask", writable && state.scenes.length > 0 && Boolean($("policyPreset")?.value), "请先选择兼容真实控制器的场景");
setActionAvailability("createCollectTask", writable && collectionReadyScenes().length > 0 && Boolean($("policyPreset")?.value), "请先保存通过运行时校验的 RM65 组合场景");
setActionAvailability("createTrainTask", writable && collectionDatasets().length > 0 && Boolean($("trainRecipe")?.value), "请先完成真实专家采集并确认 ACT 运行时");
setActionAvailability("createInferTask", writable && state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型");
setActionAvailability("addSceneAsset", writable && state.scenes.length > 0, "请先完成场景重建");
setActionAvailability("saveSceneComposition", writable && Boolean(state.composition.baseSceneId && state.composition.robotId), "请先选择基础场景和机器人");
}
function setActionAvailability(id, enabled, reason) {
@ -192,16 +214,19 @@ function switchTab(tabId) {
document.querySelectorAll(".tab").forEach((button) => button.classList.toggle("active", button.dataset.tab === tabId));
document.querySelectorAll(".page").forEach((page) => page.classList.toggle("active", page.id === tabId));
refreshSelectionBars();
if (tabId === "compose") renderComposition();
}
function renderSelectors() {
fillSelect("prepareDataset", state.datasets, "暂无数据集,请先新增");
fillSelect("reconstructInput", preparedDatasets(), "暂无已准备帧数据,请先在数据管理中准备");
fillSelect("collectScene", state.scenes, "暂无场景,请先重建");
fillSelect("collectScene", collectionReadyScenes(), "暂无可采集组合场景,请先完成场景组合");
fillSelect("trainDataset", collectionDatasets(), "暂无采集数据,请先采集");
fillSelect("inferScene", state.scenes, "暂无场景,请先重建");
fillSelect("inferModel", state.models, "暂无模型,请先训练");
fillSelect("assetScene", state.scenes, "暂无场景,请先重建");
fillSelect("composeBaseScene", baseCompositionScenes(), "暂无可编辑场景,请先导入或重建");
if ($("composeBaseScene") && state.composition.baseSceneId) $("composeBaseScene").value = state.composition.baseSceneId;
const assetScene = $("assetScene");
if (assetScene && state.activeSceneId && state.scenes.some((scene) => scene.id === state.activeSceneId)) {
assetScene.value = state.activeSceneId;
@ -233,6 +258,7 @@ function renderOverview() {
["数据导入", state.datasets.length > 0],
["输入准备", state.datasets.some((item) => item.source_type === "prepared_media")],
["场景重建", state.scenes.length > 0],
["场景组合", collectionReadyScenes().length > 0],
["数据采集", collectionDatasets().length > 0],
["模型训练", state.models.length > 0],
["在线推理", state.tasks.some((task) => task.type === "inference" && task.status === "succeeded")],
@ -262,9 +288,10 @@ function nextAction() {
if (!state.datasets.length) return { index: 0, label: "新增数据", text: "先导入 HuggingFace 样例、上传文件或本地目录。", action: () => switchTab("data") };
if (!state.datasets.some((item) => item.source_type === "prepared_media")) return { index: 1, label: "准备输入", text: "已有数据集,下一步是抽帧并生成 prepared frames。", action: () => switchTab("data") };
if (!state.scenes.length) return { index: 2, label: "开始重建", text: "已有准备数据,建议先预检,再启动真实重建。", action: () => switchTab("reconstruct") };
if (!collectionDatasets().length) return { index: 3, label: "采集数据", text: "已有场景,可用专家策略采集训练数据。", action: () => switchTab("collect") };
if (!state.models.length) return { index: 4, label: "训练模型", text: "已有采集数据,可训练策略模型。", action: () => switchTab("train") };
return { index: 5, label: "运行推理", text: "闭环资源已就绪,可以选择模型和场景在线推理。", action: () => switchTab("infer") };
if (!collectionReadyScenes().length) return { index: 3, label: "组合场景", text: "在重建场景中加入 RM65、任务资产和 MotrixSim 物理属性。", action: () => switchTab("compose") };
if (!collectionDatasets().length) return { index: 4, label: "采集数据", text: "组合场景已通过运行时校验,可以启动真实专家采集。", action: () => switchTab("collect") };
if (!state.models.length) return { index: 5, label: "训练模型", text: "已有采集数据,可训练策略模型。", action: () => switchTab("train") };
return { index: 6, label: "运行推理", text: "闭环资源已就绪,可以选择模型和场景在线推理。", action: () => switchTab("infer") };
}
function renderDatasetList() {
@ -327,6 +354,373 @@ function renderSceneList() {
setHtml("sceneList", rows || emptyState("暂无场景,请先在“场景重建”中创建。"));
}
function baseCompositionScenes() {
return state.scenes.filter((scene) => scene?.metadata?.editable !== false);
}
function collectionReadyScenes() {
return state.scenes.filter((scene) => scene?.metadata?.collection_ready || (scene.tags || []).includes("collection-ready"));
}
function ensureCompositionDraft() {
const draft = state.composition;
const scenes = baseCompositionScenes();
if (!scenes.some((scene) => scene.id === draft.baseSceneId)) {
draft.baseSceneId = scenes[0]?.id || "";
draft.bounds = null;
draft.previewSceneId = "";
}
if (!state.robots.some((robot) => robot.id === draft.robotId)) {
const robot = state.robots[0];
draft.robotId = robot?.id || "";
draft.robotTransform = robot ? cloneValue(robot.default_transform) : null;
}
const validIds = new Set([`robot:${draft.robotId}`, ...draft.assets.map((asset) => asset.instance_id)]);
if (!validIds.has(draft.selectedId)) draft.selectedId = draft.robotId ? `robot:${draft.robotId}` : draft.assets[0]?.instance_id || "";
}
function renderComposition() {
if (!$("composeRobotList")) return;
ensureCompositionDraft();
const draft = state.composition;
const baseScene = state.scenes.find((scene) => scene.id === draft.baseSceneId);
const baseSelect = $("composeBaseScene");
if (baseSelect && baseScene) baseSelect.value = baseScene.id;
setHtml("composeRobotList", state.robots.map((robot) => `
<button class="composition-resource-card ${robot.id === draft.robotId ? "active" : ""}" type="button" draggable="true" data-compose-robot="${escapeAttr(robot.id)}">
<span class="resource-mark">RM65</span>
<span><strong>${escapeHtml(robot.name)}</strong><span>${escapeHtml(robot.description || "MotrixSim 机器人")}</span><em>${robot.id === draft.robotId ? "已加入场景" : "拖入场景"}</em></span>
</button>`).join("") || emptyState("暂无已注册机器人。"));
const visibleAssets = state.digitalAssets.filter((asset) => asset.source_type === "recipe" || asset.source_type === "primitive" || asset.capabilities?.browser_preview).slice(0, 30);
setHtml("composeAssetList", visibleAssets.map((asset) => {
const inScene = draft.assets.some((item) => item.catalog_id === asset.id);
return `<button class="composition-asset-card" type="button" draggable="true" data-compose-catalog="${escapeAttr(asset.id)}" data-in-scene="${inScene}">
<span><strong>${escapeHtml(asset.name)}</strong><span>${escapeHtml(asset.description || compositionAssetDescription(asset))}</span></span>
<em>${inScene && asset.source_type === "recipe" ? "已在场景中" : "拖入场景"}</em>
</button>`;
}).join("") || emptyState("暂无可用于组合的数字资产。"));
const hierarchy = [
baseScene ? { id: "base-scene", name: resourceDisplayName(baseScene), type: "基础场景", removable: false } : null,
draft.robotId ? { id: `robot:${draft.robotId}`, name: state.robots.find((robot) => robot.id === draft.robotId)?.name || draft.robotId, type: "机器人", removable: true } : null,
...draft.assets.map((asset) => ({ id: asset.instance_id, name: asset.name, type: compositionAssetType(asset), removable: true })),
].filter(Boolean);
setText("compositionObjectCount", `${hierarchy.length} 个对象`);
setHtml("compositionHierarchy", hierarchy.map((item) => `
<div class="composition-tree-row ${item.id === draft.selectedId ? "active" : ""}" data-compose-select="${escapeAttr(item.id)}">
<strong>${escapeHtml(item.name)}</strong><span>${escapeHtml(item.type)}</span>${item.removable ? `<button class="danger-button" type="button" data-compose-remove="${escapeAttr(item.id)}">移除</button>` : "<span></span>"}
</div>`).join("") || emptyState("先选择基础场景和机器人。"));
renderCompositionInspector();
renderCompositionPreview(baseScene);
const requiredReady = ["recipe:coffee_table", "recipe:pick_cube", "recipe:place_target"].every((id) => draft.assets.some((asset) => asset.catalog_id === id));
const pending = state.tasks.find((task) => task.id === draft.pendingTaskId);
if (pending?.status === "succeeded" && pending.result?.scene_id) {
draft.savedSceneId = pending.result.scene_id;
draft.collectionReady = Boolean(pending.result.collection_ready);
draft.pendingTaskId = "";
}
const savedScene = state.scenes.find((scene) => scene.id === draft.savedSceneId);
const motrixReady = Boolean(state.capabilities?.runtimes?.motrixsim?.ready);
setHtml("composeCompatibility", `
<strong>${motrixReady ? "MotrixSim 运行时已就绪" : "MotrixSim 运行时不可用"}</strong>
<span>RM65:${draft.robotId ? "已加入" : "未选择"}</span>
<span>采集对象:${requiredReady ? "茶几、抓取方块、放置目标齐全" : "还需加入茶几、抓取方块和放置目标"}</span>
<span>保存状态:${savedScene ? `已生成 ${escapeHtml(resourceDisplayName(savedScene))}` : pending ? escapeHtml(statusLabel(pending.status)) : "尚未保存"}</span>`);
setText("composeStageStatus", baseScene ? `${resourceDisplayName(baseScene)} · ${draft.assets.length} 个数字资产 · ${draft.robotId ? "RM65 已加入" : "未选择机器人"}` : "等待基础场景");
setActionAvailability("saveSceneComposition", state.user?.role !== "viewer" && Boolean(baseScene && draft.robotId && motrixReady), motrixReady ? "请先选择基础场景和机器人" : "MotrixSim worker 尚未就绪");
setActionAvailability("testSceneComposition", Boolean(savedScene), "请先保存并等待组合场景通过运行时校验");
setActionAvailability("collectSceneComposition", Boolean(savedScene && draft.collectionReady), "组合场景需包含茶几、抓取方块和放置目标并通过校验");
}
function renderCompositionPreview(baseScene) {
const iframe = $("compositionPreview");
if (!iframe) return;
if (!baseScene) {
iframe.removeAttribute("src");
return;
}
if (state.composition.previewSceneId !== baseScene.id || !iframe.src.includes(STATIC_VERSION)) {
state.composition.previewSceneId = baseScene.id;
iframe.src = `/static/scene-viewer.html?v=${STATIC_VERSION}&embedded=1&composer=1&scene_id=${encodeURIComponent(baseScene.id)}`;
iframe.onload = () => postCompositionDraft();
loadCompositionBounds(baseScene.id).catch((error) => console.warn(error));
} else {
postCompositionDraft();
}
}
async function loadCompositionBounds(sceneId) {
const manifest = await api(`/api/scenes/${encodeURIComponent(sceneId)}/manifest`);
if (state.composition.baseSceneId !== sceneId) return;
const min = Array.isArray(manifest.bounds_m?.min) ? manifest.bounds_m.min.map(Number) : [];
const max = Array.isArray(manifest.bounds_m?.max) ? manifest.bounds_m.max.map(Number) : [];
state.composition.bounds = min.length === 3 && max.length === 3 && [...min, ...max].every(Number.isFinite) ? { min, max } : null;
}
function postCompositionDraft() {
const iframe = $("compositionPreview");
if (!iframe?.contentWindow) return;
const draft = state.composition;
const robot = state.robots.find((item) => item.id === draft.robotId);
const robotPayload = robot ? {
...robot,
instance_id: `robot:${robot.id}`,
role: "robot",
kind: "robot",
name: "rm65.glb",
url: robot.preview_url,
transform: cloneValue(draft.robotTransform),
} : null;
const assets = draft.assets.map((instance) => {
const catalog = state.digitalAssets.find((item) => item.id === instance.catalog_id) || {};
const components = Array.isArray(catalog.components) ? catalog.components.map((component) => ({
name: component.name,
viewer_primitive: component.primitive,
translation: component.position || [0, 0, 0],
rotation_euler_degrees: component.rotation_euler_degrees || [0, 0, 0],
dimensions: component.dimensions || catalog.physics?.dimensions || [0.1, 0.1, 0.1],
rgba: component.rgba || catalog.physics?.rgba || [0.13, 0.76, 0.67, 1],
})) : [];
return {
...catalog,
...cloneValue(instance),
role: "composition_draft",
url: catalog.preview_url || catalog.url || "",
viewer_primitive: components.length === 1 ? components[0].viewer_primitive : catalog.viewer_primitive,
viewer_components: components,
};
});
iframe.contentWindow.postMessage({
type: "real2sim-composition-draft",
robot: robotPayload,
assets,
selected_id: draft.selectedId,
tool: draft.tool,
}, location.origin);
}
function selectedCompositionObject() {
const draft = state.composition;
if (draft.selectedId === `robot:${draft.robotId}`) {
const robot = state.robots.find((item) => item.id === draft.robotId);
return robot ? { type: "robot", id: draft.selectedId, name: robot.name, transform: draft.robotTransform, robot } : null;
}
const asset = draft.assets.find((item) => item.instance_id === draft.selectedId);
return asset ? { type: "asset", id: asset.instance_id, name: asset.name, transform: asset.transform, physics: asset.physics, asset } : null;
}
function renderCompositionInspector() {
const selected = selectedCompositionObject();
setHtml("compositionSelection", selected
? `<strong>${escapeHtml(selected.name)}</strong><span>${selected.type === "robot" ? "RM65 机器人 · 关节与惯性保持标定尺度" : escapeHtml(compositionAssetType(selected.asset))}</span>`
: "选择画布或层级中的对象");
const transform = selected?.transform || { translation: [0, 0, 0], rotation_euler_degrees: [0, 0, 0], scale: [1, 1, 1] };
const values = {
composePosX: transform.translation?.[0] ?? 0,
composePosY: transform.translation?.[1] ?? 0,
composePosZ: transform.translation?.[2] ?? 0,
composeRotX: transform.rotation_euler_degrees?.[0] ?? 0,
composeRotY: transform.rotation_euler_degrees?.[1] ?? 0,
composeRotZ: transform.rotation_euler_degrees?.[2] ?? 0,
composeScaleX: transform.scale?.[0] ?? 1,
composeScaleY: transform.scale?.[1] ?? 1,
composeScaleZ: transform.scale?.[2] ?? 1,
composeMass: selected?.physics?.mass ?? selected?.robot?.payload_kg ?? 0,
};
Object.entries(values).forEach(([id, value]) => {
const input = $(id);
if (input && document.activeElement !== input) input.value = Number(value).toFixed(id.startsWith("composeRot") ? 1 : 3);
});
if ($("composeDynamic")) $("composeDynamic").checked = Boolean(selected?.physics?.dynamic);
if ($("composeCollision")) $("composeCollision").value = selected?.physics?.collision_mode || "visual_and_collision";
const disabled = !selected;
["composePosX", "composePosY", "composePosZ", "composeRotX", "composeRotY", "composeRotZ"].forEach((id) => { if ($(id)) $(id).disabled = disabled; });
["composeScaleX", "composeScaleY", "composeScaleZ", "composeDynamic", "composeMass", "composeCollision"].forEach((id) => { if ($(id)) $(id).disabled = disabled || selected?.type === "robot"; });
if ($("removeCompositionObject")) $("removeCompositionObject").disabled = !selected;
}
function compositionAssetDescription(asset) {
if (asset.source_type === "primitive") return `${sourceTypeLabel(asset.format)} · 可设置碰撞和动态属性`;
if (asset.source_type === "dataset") return `${asset.dataset_name || "数据集"} · ${String(asset.format || "mesh").toUpperCase()}`;
return "MotrixSim 数字资产";
}
function compositionAssetType(asset) {
const catalog = state.digitalAssets.find((item) => item.id === asset?.catalog_id);
if (catalog?.source_type === "recipe") return catalog.category === "task_object" ? "动态任务物体" : catalog.category === "task_marker" ? "任务目标" : "组合资产";
return catalog?.source_type === "dataset" ? "已导入网格" : "基础几何";
}
function cloneValue(value) {
return value == null ? value : JSON.parse(JSON.stringify(value));
}
function compositionSnapshot() {
const draft = state.composition;
return cloneValue({ robotId: draft.robotId, robotTransform: draft.robotTransform, assets: draft.assets, selectedId: draft.selectedId });
}
function pushCompositionHistory() {
const draft = state.composition;
draft.history.push(compositionSnapshot());
if (draft.history.length > 40) draft.history.shift();
draft.future = [];
}
function restoreCompositionSnapshot(snapshot) {
if (!snapshot) return;
Object.assign(state.composition, cloneValue(snapshot), { savedSceneId: "", collectionReady: false, pendingTaskId: "" });
renderComposition();
}
function markCompositionDirty() {
Object.assign(state.composition, { savedSceneId: "", collectionReady: false, pendingTaskId: "" });
}
function addCompositionAsset(catalogId, position = null, { record = true } = {}) {
const catalog = state.digitalAssets.find((asset) => asset.id === catalogId);
if (!catalog) return toast("所选数字资产已不在资源目录中");
if (catalog.source_type === "recipe") {
const existing = state.composition.assets.find((asset) => asset.catalog_id === catalogId);
if (existing) {
state.composition.selectedId = existing.instance_id;
renderComposition();
return toast("该任务资产已在场景中,可直接调整位置");
}
}
if (record) pushCompositionHistory();
const transform = cloneValue(catalog.default_transform || { translation: [0, 0, 0.1], rotation_euler_degrees: [0, 0, 0], scale: [1, 1, 1] });
if (position) {
transform.translation[0] = position[0];
transform.translation[1] = position[1];
}
const instance = {
catalog_id: catalog.id,
instance_id: `asset_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 7)}`,
name: catalog.name,
source: cloneValue(catalog.source || {}),
transform,
physics: cloneValue(catalog.physics || { dynamic: false, mass: 0.25, collision_mode: "visual_and_collision", dimensions: [0.1, 0.1, 0.1] }),
};
state.composition.assets.push(instance);
state.composition.selectedId = instance.instance_id;
markCompositionDirty();
renderComposition();
}
function compositionDropPosition(event) {
const bounds = state.composition.bounds;
const catalog = state.digitalAssets.find((asset) => asset.id === document.body.dataset.compositionCatalog);
const fallback = cloneValue(catalog?.default_transform?.translation || [0, 0, 0.1]);
if (!bounds) return fallback;
const rect = $("compositionDropOverlay").getBoundingClientRect();
const nx = Math.max(0, Math.min(1, (event.clientX - rect.left) / Math.max(1, rect.width)));
const ny = Math.max(0, Math.min(1, (event.clientY - rect.top) / Math.max(1, rect.height)));
const width = Math.min(6, Math.max(0.4, bounds.max[0] - bounds.min[0]));
const depth = Math.min(6, Math.max(0.4, bounds.max[1] - bounds.min[1]));
const centerX = (bounds.min[0] + bounds.max[0]) / 2;
const centerY = (bounds.min[1] + bounds.max[1]) / 2;
return [centerX + (0.5 - ny) * width * 0.7, centerY + (nx - 0.5) * depth * 0.7, fallback[2]];
}
function selectCompositionObject(id) {
state.composition.selectedId = id;
renderComposition();
}
function removeCompositionObject(id = state.composition.selectedId) {
if (!id) return;
pushCompositionHistory();
if (id.startsWith("robot:")) {
state.composition.robotId = "";
state.composition.robotTransform = null;
} else {
state.composition.assets = state.composition.assets.filter((asset) => asset.instance_id !== id);
}
state.composition.selectedId = state.composition.robotId ? `robot:${state.composition.robotId}` : state.composition.assets[0]?.instance_id || "";
markCompositionDirty();
renderComposition();
}
function updateSelectedCompositionFromForm() {
const selected = selectedCompositionObject();
if (!selected) return;
pushCompositionHistory();
selected.transform.translation = [Number($("composePosX").value), Number($("composePosY").value), Number($("composePosZ").value)];
selected.transform.rotation_euler_degrees = [Number($("composeRotX").value), Number($("composeRotY").value), Number($("composeRotZ").value)];
if (selected.type !== "robot") {
selected.transform.scale = [Number($("composeScaleX").value), Number($("composeScaleY").value), Number($("composeScaleZ").value)];
selected.physics.dynamic = Boolean($("composeDynamic").checked);
selected.physics.mass = Number($("composeMass").value);
selected.physics.collision_mode = $("composeCollision").value;
}
markCompositionDirty();
renderComposition();
}
function loadPickPlacePreset() {
pushCompositionHistory();
for (const catalog of state.digitalAssets.filter((asset) => asset.source_type === "recipe" && asset.required_for_collection)) {
if (!state.composition.assets.some((asset) => asset.catalog_id === catalog.id)) addCompositionAsset(catalog.id, null, { record: false });
}
markCompositionDirty();
renderComposition();
}
async function saveSceneComposition() {
const draft = state.composition;
if (!draft.baseSceneId || !draft.robotId) return toast("请先选择基础场景和机器人");
const name = $("compositionName")?.value.trim() || "RM65 组合场景";
setBusy(true, "正在创建 MotrixSim 场景组合任务");
try {
const result = await api(`/api/scenes/${encodeURIComponent(draft.baseSceneId)}/compose`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
name,
robot: { robot_id: draft.robotId, transform: cloneValue(draft.robotTransform) },
assets: draft.assets.map((asset) => ({
catalog_id: asset.catalog_id,
instance_id: asset.instance_id,
name: asset.name,
source: cloneValue(asset.source || {}),
transform: cloneValue(asset.transform),
physics: cloneValue(asset.physics || {}),
})),
validation_steps: 120,
}),
});
draft.pendingTaskId = result.task.id;
draft.savedSceneId = "";
draft.collectionReady = false;
state.selectedTaskId = result.task.id;
toast("组合任务已创建,正在执行真实 MotrixSim 校验");
await refresh({ quiet: true });
showTask(result.task.id).catch((error) => toast(error.message));
} finally {
setBusy(false);
}
}
async function testSceneComposition() {
const sceneId = state.composition.savedSceneId;
if (!sceneId) return toast("请先保存组合场景");
await createTask("simulator_validation", { scene_id: sceneId, simulators: ["mujoco", "motrixsim"], level: "runtime_required" });
switchTab("compose");
}
function collectSceneComposition() {
const sceneId = state.composition.savedSceneId;
if (!sceneId || !state.composition.collectionReady) return toast("该组合场景尚未满足采集对象约定");
switchTab("collect");
renderSelectors();
if ($("collectScene")) $("collectScene").value = sceneId;
toast("已选择组合场景,可配置 Episode 数后开始采集");
}
function renderPreflightSummary() {
const node = $("preflightSummary");
if (!node) return;
@ -1091,7 +1485,7 @@ function collectionDatasets() {
}
function sceneIdFromTask(task) {
return ["scene_reconstruction", "motrixsim_scene_import", "scene_asset_compose"].includes(task?.type) ? task.result?.scene_id || "" : "";
return ["scene_reconstruction", "motrixsim_scene_import", "scene_asset_compose", "scene_composition"].includes(task?.type) ? task.result?.scene_id || "" : "";
}
function resourceView(kind, item) {
@ -1187,7 +1581,7 @@ function taskSubtitle(task) {
function taskProgress(task) {
if (task.status === "succeeded") return 100;
if (Number.isFinite(Number(task.progress))) return Math.max(0, Math.min(100, Number(task.progress)));
return { succeeded: 100, running: 50, queued: 8, canceling: 45 }[task.status] || 0;
return 0;
}
function taskStage(task) {
@ -1205,6 +1599,7 @@ function taskTypeLabel(type) {
motrixsim_asset_import: "MotrixSim 资产导入",
motrixsim_scene_import: "MotrixSim 场景包导入",
scene_asset_compose: "数字资产合成",
scene_composition: "场景组合",
simulator_validation: "仿真校验",
renderer_benchmark: "渲染器对比",
scene_cover_render: "场景封面",
@ -1253,6 +1648,9 @@ function sourceTypeLabel(type) {
scene_reconstruction: "场景重建",
motrixsim_spatial_bundle: "MotrixSim 空间包",
scene_revision: "场景修订",
scene_composition: "场景组合",
recipe: "任务资产配方",
primitive: "基础几何",
training: "训练模型",
inference: "推理结果",
splat_or_pointcloud: "点云 / Splat",
@ -1377,7 +1775,7 @@ function assetPriority(asset) {
}
function assetRoleLabel(role) {
return ({ added_asset: "新增资产", collision: "碰撞体", visual: "可视化", texture: "纹理", simulator_import: "仿真导入", package: "场景包", source: "来源资产" })[role] || role;
return ({ robot: "机器人", added_asset: "新增资产", collision: "碰撞体", visual: "可视化", texture: "纹理", simulator_import: "仿真导入", package: "场景包", source: "来源资产" })[role] || role;
}
function bulkMessage(result, label) {
@ -1396,6 +1794,7 @@ function exportSummary(payload = {}) {
function switchTaskPage(type) {
if (["hf_import", "media_prepare"].includes(type)) switchTab("data");
else if (type === "scene_composition") switchTab("compose");
else if (["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"].includes(type)) switchTab("reconstruct");
else if (type === "expert_collection") switchTab("collect");
else if (type === "training") switchTab("train");
@ -1599,6 +1998,43 @@ function bindEvents() {
$("reconstructInput")?.addEventListener("change", () => {
renderReconstructDatasetCards();
});
$("composeBaseScene")?.addEventListener("change", () => {
pushCompositionHistory();
state.composition.baseSceneId = $("composeBaseScene").value;
state.composition.bounds = null;
state.composition.previewSceneId = "";
markCompositionDirty();
renderComposition();
});
$("loadPickPlacePreset")?.addEventListener("click", loadPickPlacePreset);
$("saveSceneComposition")?.addEventListener("click", saveSceneComposition);
$("testSceneComposition")?.addEventListener("click", testSceneComposition);
$("collectSceneComposition")?.addEventListener("click", collectSceneComposition);
$("removeCompositionObject")?.addEventListener("click", () => removeCompositionObject());
["composePosX", "composePosY", "composePosZ", "composeRotX", "composeRotY", "composeRotZ", "composeScaleX", "composeScaleY", "composeScaleZ", "composeDynamic", "composeMass", "composeCollision"].forEach((id) => {
$(id)?.addEventListener("change", updateSelectedCompositionFromForm);
});
$("composeUndo")?.addEventListener("click", () => {
const snapshot = state.composition.history.pop();
if (!snapshot) return toast("没有可撤销的操作");
state.composition.future.push(compositionSnapshot());
restoreCompositionSnapshot(snapshot);
});
$("composeRedo")?.addEventListener("click", () => {
const snapshot = state.composition.future.pop();
if (!snapshot) return toast("没有可重做的操作");
state.composition.history.push(compositionSnapshot());
restoreCompositionSnapshot(snapshot);
});
$("compositionDropOverlay")?.addEventListener("dragover", (event) => event.preventDefault());
$("compositionDropOverlay")?.addEventListener("drop", (event) => {
event.preventDefault();
const catalogId = document.body.dataset.compositionCatalog;
const robotId = document.body.dataset.compositionRobot;
if (catalogId) addCompositionAsset(catalogId, compositionDropPosition(event));
if (robotId) selectCompositionRobot(robotId, compositionDropPosition(event));
clearCompositionDrag();
});
$("createMediaPrepare")?.addEventListener("click", async () => {
const datasetId = $("prepareDataset")?.value || state.datasets[0]?.id;
@ -1649,8 +2085,8 @@ function bindEvents() {
});
$("createCollectTask")?.addEventListener("click", async () => {
const sceneId = $("collectScene")?.value || state.scenes[0]?.id;
if (!sceneId) return toast("请先完成场景重建。");
const sceneId = $("collectScene")?.value || collectionReadyScenes()[0]?.id;
if (!sceneId) return toast("请先保存满足 RM65 控制器约定的组合场景。");
await createTask("expert_collection", {
scene_id: sceneId,
policy_preset: $("policyPreset")?.value,
@ -1692,10 +2128,93 @@ function bindEvents() {
}
});
document.body.addEventListener("dragstart", (event) => {
const asset = event.target.closest("[data-compose-catalog]");
const robot = event.target.closest("[data-compose-robot]");
if (!asset && !robot) return;
document.body.dataset.compositionCatalog = asset?.dataset.composeCatalog || "";
document.body.dataset.compositionRobot = robot?.dataset.composeRobot || "";
document.body.classList.add("composition-dragging");
event.dataTransfer.effectAllowed = "copy";
event.dataTransfer.setData("text/plain", asset?.dataset.composeCatalog || robot?.dataset.composeRobot || "real2sim-resource");
});
document.body.addEventListener("dragend", clearCompositionDrag);
document.body.addEventListener("click", handleBodyClick);
window.addEventListener("message", handleCompositionViewerMessage);
}
function clearCompositionDrag() {
document.body.classList.remove("composition-dragging");
delete document.body.dataset.compositionCatalog;
delete document.body.dataset.compositionRobot;
}
function selectCompositionRobot(robotId, position = null) {
const robot = state.robots.find((item) => item.id === robotId);
if (!robot) return toast("机器人已不在资源目录中");
pushCompositionHistory();
state.composition.robotId = robot.id;
state.composition.robotTransform = cloneValue(robot.default_transform);
if (position) {
state.composition.robotTransform.translation[0] = position[0];
state.composition.robotTransform.translation[1] = position[1];
}
state.composition.selectedId = `robot:${robot.id}`;
markCompositionDirty();
renderComposition();
}
function handleCompositionViewerMessage(event) {
const iframe = $("compositionPreview");
if (event.origin !== location.origin || event.source !== iframe?.contentWindow || !event.data) return;
const payload = event.data;
if (payload.type === "real2sim-composition-selected") {
state.composition.selectedId = String(payload.instance_id || "");
renderCompositionInspector();
return;
}
if (payload.type === "real2sim-composition-transform-start") {
pushCompositionHistory();
return;
}
if (payload.type === "real2sim-composition-transform") {
const id = String(payload.instance_id || "");
const transform = cloneValue(payload.transform || {});
if (id === `robot:${state.composition.robotId}`) state.composition.robotTransform = transform;
else {
const asset = state.composition.assets.find((item) => item.instance_id === id);
if (asset) asset.transform = transform;
}
state.composition.selectedId = id;
markCompositionDirty();
renderCompositionInspector();
setText("composeStageStatus", "对象位置已更新,保存后将写入 MotrixSim MJCF");
return;
}
if (payload.type === "real2sim-composition-transform-end") renderComposition();
}
async function handleBodyClick(event) {
const composeRobot = event.target.closest("[data-compose-robot]");
if (composeRobot) selectCompositionRobot(composeRobot.dataset.composeRobot);
const composeCatalog = event.target.closest("[data-compose-catalog]");
if (composeCatalog) addCompositionAsset(composeCatalog.dataset.composeCatalog);
const composeSelect = event.target.closest("[data-compose-select]");
if (composeSelect && !event.target.closest("button")) selectCompositionObject(composeSelect.dataset.composeSelect);
const composeRemove = event.target.closest("[data-compose-remove]");
if (composeRemove) removeCompositionObject(composeRemove.dataset.composeRemove);
const composeTool = event.target.closest("[data-compose-tool]");
if (composeTool) {
state.composition.tool = composeTool.dataset.composeTool;
document.querySelectorAll("[data-compose-tool]").forEach((button) => button.classList.toggle("active", button === composeTool));
postCompositionDraft();
}
const closeDialog = event.target.closest("[data-close-dialog]");
if (closeDialog) $(closeDialog.dataset.closeDialog)?.close();

Binary file not shown.

View File

@ -52,6 +52,7 @@
<button class="tab" data-tab="data">数据管理</button>
<button class="tab" data-tab="reconstruct">场景重建</button>
<button class="tab" data-tab="scenes">场景管理</button>
<button class="tab" data-tab="compose">场景组合</button>
<button class="tab" data-tab="collect">数据采集</button>
<button class="tab" data-tab="train">模型训练</button>
<button class="tab" data-tab="infer">在线推理</button>
@ -386,6 +387,111 @@
</section>
</section>
<section id="compose" class="page">
<div class="page-head">
<div>
<h1>场景组合</h1>
<p>选择基础场景与机器人,将真实数字资产拖入三维画布;保存后生成可测试、可采集的 MotrixSim 场景版本。</p>
</div>
<button id="loadPickPlacePreset" type="button">加载 Pick-Place 预设</button>
</div>
<div class="composition-workbench">
<section class="panel composition-library">
<div class="panel-head">
<div>
<h2>资源库</h2>
<p>资源来自平台目录,拖入中间画布进行布置</p>
</div>
</div>
<div class="composition-library-body">
<label>基础场景<select id="composeBaseScene"></select></label>
<div class="composition-section-head"><strong>机器人</strong><span>选择一个 MotrixSim 机器人</span></div>
<div id="composeRobotList" class="composition-resource-list"></div>
<div class="composition-section-head"><strong>数字资产</strong><span>拖动或点击加入场景</span></div>
<div id="composeAssetList" class="composition-asset-grid"></div>
</div>
</section>
<section class="panel composition-stage-panel">
<div class="composition-toolbar" role="toolbar" aria-label="场景组合工具">
<button class="active" type="button" data-compose-tool="select">选择</button>
<button type="button" data-compose-tool="move">移动</button>
<button type="button" data-compose-tool="rotate">旋转</button>
<button type="button" data-compose-tool="scale">缩放</button>
<span class="toolbar-separator"></span>
<button id="composeUndo" type="button">撤销</button>
<button id="composeRedo" type="button">重做</button>
<span id="composeStageStatus" class="composition-stage-status">等待基础场景</span>
</div>
<div id="compositionCanvasWrap" class="composition-canvas-wrap">
<iframe id="compositionPreview" title="MotrixSim 场景组合编辑器"></iframe>
<div id="compositionDropOverlay" class="composition-drop-overlay">释放以加入场景</div>
</div>
<div class="composition-hierarchy-head"><strong>对象层级</strong><span id="compositionObjectCount">0 个对象</span></div>
<div id="compositionHierarchy" class="composition-hierarchy"></div>
</section>
<section class="panel composition-inspector">
<div class="panel-head">
<div>
<h2>属性检查器</h2>
<p>修改选中对象的坐标与 MotrixSim 物理属性</p>
</div>
<button id="removeCompositionObject" class="danger-button" type="button">移除</button>
</div>
<div class="composition-inspector-body">
<div id="compositionSelection" class="composition-selection-card">选择画布或层级中的对象</div>
<fieldset>
<legend>变换</legend>
<div class="axis-grid">
<label>位置 X<input id="composePosX" type="number" step="0.01"></label>
<label>位置 Y<input id="composePosY" type="number" step="0.01"></label>
<label>位置 Z<input id="composePosZ" type="number" step="0.01"></label>
<label>旋转 X<input id="composeRotX" type="number" step="1"></label>
<label>旋转 Y<input id="composeRotY" type="number" step="1"></label>
<label>旋转 Z<input id="composeRotZ" type="number" step="1"></label>
<label>缩放 X<input id="composeScaleX" type="number" min="0.01" step="0.01"></label>
<label>缩放 Y<input id="composeScaleY" type="number" min="0.01" step="0.01"></label>
<label>缩放 Z<input id="composeScaleZ" type="number" min="0.01" step="0.01"></label>
</div>
</fieldset>
<fieldset>
<legend>物理属性</legend>
<label class="toggle-field"><input id="composeDynamic" type="checkbox"> 动态物体</label>
<label>质量(kg)<input id="composeMass" type="number" min="0.001" step="0.01"></label>
<label>碰撞模式<select id="composeCollision"><option value="visual_and_collision">可见并参与碰撞</option><option value="visual_only">仅可见</option><option value="collision_only">仅碰撞</option></select></label>
</fieldset>
<div id="composeCompatibility" class="compatibility-card"></div>
</div>
</section>
</div>
<div class="composition-action-bar">
<label>组合场景名称<input id="compositionName" value="RM65 茶几 Pick-Place"></label>
<button id="saveSceneComposition" class="primary-button" type="button">保存为组合场景</button>
<button id="testSceneComposition" type="button" disabled>运行 MotrixSim 测试</button>
<button id="collectSceneComposition" type="button" disabled>进入数据采集</button>
</div>
<section class="panel composition-tasks-panel">
<div class="panel-head">
<div>
<h2>组合与测试任务</h2>
<p>保存、运行时校验和失败原因均显示在这里</p>
</div>
</div>
<div class="bulk-bar" data-bulk-scope="compositionTaskList">
<label><input type="checkbox" data-select-all-tasks="compositionTaskList"> 全选</label>
<span data-selected-count="compositionTaskList">已选择 0 项</span>
<button data-bulk-task-action="retry">批量重试</button>
<button data-bulk-task-action="cancel">批量取消</button>
<button class="danger-button" data-bulk-task-action="delete">批量删除</button>
</div>
<div id="compositionTaskList" class="list-stack"></div>
</section>
</section>
<section id="collect" class="page">
<div class="page-head">
<div>

View File

@ -72,10 +72,11 @@
const params = new URLSearchParams(location.search);
const sceneId = params.get("scene_id");
const embeddedMode = params.get("embedded") === "1";
const composerMode = params.get("composer") === "1";
const coverMode = params.get("cover") === "1";
const coverView = params.get("cover_view") || "hero";
const probeFull = params.get("probe_full") === "1";
if (coverMode) document.getElementById("hud").style.display = "none";
if (coverMode || composerMode) document.getElementById("hud").style.display = "none";
setupHudPanel();
if (embeddedMode || innerWidth <= 640) setHudCollapsed(true);
const nativeWarn = console.warn.bind(console);
@ -118,6 +119,11 @@
let loadedObject = null;
let overlayObjects = [];
const draftObjects = new Map();
let selectedDraftId = "";
let selectionHelper = null;
let composerTool = "select";
let compositionDrag = null;
let manifestAssets = [];
let manifestBounds = null;
let sourceUpAxis = "Z";
@ -136,6 +142,7 @@
};
window.real2simViewerDebug = debug;
window.real2simViewerDebug.snapshot = () => ({ ...debug, canvas: { ...debug.canvas } });
if (composerMode) setupCompositionEditor();
function setupHudPanel() {
let drag = null;
@ -224,7 +231,7 @@
return;
}
const selected = assets[Number(button.dataset.index)];
if (selected?.role === "added_asset") {
if (["added_asset", "robot"].includes(selected?.role)) {
status.textContent = `已添加资产 · ${selected.name || "asset"} · 与 MotrixSim 场景使用同一坐标`;
return;
}
@ -528,7 +535,7 @@
async function loadSceneOverlays(assets) {
clearOverlays();
if (!loadedObject) return;
const overlays = assets.filter((asset) => asset?.role === "added_asset");
const overlays = assets.filter((asset) => ["added_asset", "robot"].includes(asset?.role));
for (const asset of overlays) {
try {
const object = await createOverlayObject(asset);
@ -548,9 +555,44 @@
}
async function createOverlayObject(asset) {
if (Array.isArray(asset.viewer_components) && asset.viewer_components.length) {
const group = new THREE.Group();
for (const component of asset.viewer_components) {
const child = createPrimitiveObject(
String(component.viewer_primitive || "box").toLowerCase(),
Array.isArray(component.dimensions) ? component.dimensions.map(Number) : [0.1, 0.1, 0.1],
Array.isArray(component.rgba) ? component.rgba.map(Number) : [0.13, 0.76, 0.67, 1],
);
if (!child) continue;
const translation = Array.isArray(component.translation) ? component.translation.map(Number) : [0, 0, 0];
const rotation = Array.isArray(component.rotation_euler_degrees) ? component.rotation_euler_degrees.map(Number) : [0, 0, 0];
child.position.set(translation[0] || 0, translation[1] || 0, translation[2] || 0);
child.rotation.set(THREE.MathUtils.degToRad(rotation[0] || 0), THREE.MathUtils.degToRad(rotation[1] || 0), THREE.MathUtils.degToRad(rotation[2] || 0));
group.add(child);
}
return group;
}
const primitive = String(asset.viewer_primitive || "").toLowerCase();
const dimensions = Array.isArray(asset.physics?.dimensions) ? asset.physics.dimensions.map(Number) : [0.1, 0.1, 0.1];
const rgba = Array.isArray(asset.physics?.rgba) ? asset.physics.rgba.map(Number) : [0.13, 0.76, 0.67, 1];
const primitiveObject = createPrimitiveObject(primitive, dimensions, rgba);
if (primitiveObject) return primitiveObject;
if (!asset.url) return null;
const suffix = String(asset.name || asset.path || asset.url || "").toLowerCase().split("?")[0].split(".").pop();
if (suffix === "glb" || suffix === "gltf") {
const { GLTFLoader } = await import("three/addons/loaders/GLTFLoader.js");
return (await new GLTFLoader().loadAsync(asset.url)).scene;
}
if (suffix === "obj") return new OBJLoader().loadAsync(asset.url);
if (suffix === "ply") {
const geometry = await new PLYLoader().loadAsync(asset.url);
geometry.computeVertexNormals();
return new THREE.Mesh(geometry, material);
}
return null;
}
function createPrimitiveObject(primitive, dimensions, rgba) {
const material = new THREE.MeshStandardMaterial({
color: new THREE.Color(rgba[0], rgba[1], rgba[2]),
transparent: rgba[3] < 1,
@ -580,18 +622,6 @@
group.add(first, second);
return group;
}
if (!asset.url) return null;
const suffix = String(asset.name || asset.path || "").toLowerCase().split(".").pop();
if (suffix === "glb" || suffix === "gltf") {
const { GLTFLoader } = await import("three/addons/loaders/GLTFLoader.js");
return (await new GLTFLoader().loadAsync(asset.url)).scene;
}
if (suffix === "obj") return new OBJLoader().loadAsync(asset.url);
if (suffix === "ply") {
const geometry = await new PLYLoader().loadAsync(asset.url);
geometry.computeVertexNormals();
return new THREE.Mesh(geometry, material);
}
return null;
}
@ -610,6 +640,158 @@
debug.overlayCount = 0;
}
function setupCompositionEditor() {
const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2();
const dragPlane = new THREE.Plane();
addEventListener("message", (event) => {
if (event.origin !== location.origin || event.data?.type !== "real2sim-composition-draft") return;
composerTool = ["select", "move", "rotate", "scale"].includes(event.data.tool) ? event.data.tool : "select";
loadCompositionDraft(event.data).catch((error) => console.error("Unable to load composition draft", error));
});
renderer.domElement.addEventListener("pointerdown", (event) => {
const hit = compositionHit(event, raycaster, pointer);
if (!hit) return;
selectDraftObject(hit.userData.compositionInstanceId, true);
if (composerTool === "select") return;
const object = draftObjects.get(hit.userData.compositionInstanceId);
if (!object) return;
compositionDrag = {
instanceId: hit.userData.compositionInstanceId,
object,
startX: event.clientX,
startY: event.clientY,
transform: cloneTransform(object.userData.compositionTransform),
};
if (composerTool === "move") {
contentRoot.updateMatrixWorld(true);
const localPoint = new THREE.Vector3(0, 0, Number(compositionDrag.transform.translation?.[2] || 0)).applyMatrix4(contentRoot.matrixWorld);
const normal = new THREE.Vector3(0, 0, 1).transformDirection(contentRoot.matrixWorld);
dragPlane.setFromNormalAndCoplanarPoint(normal, localPoint);
}
controls.enabled = false;
renderer.domElement.setPointerCapture(event.pointerId);
parent.postMessage({ type: "real2sim-composition-transform-start", instance_id: compositionDrag.instanceId }, location.origin);
event.preventDefault();
});
renderer.domElement.addEventListener("pointermove", (event) => {
if (!compositionDrag) return;
const transform = cloneTransform(compositionDrag.transform);
if (composerTool === "move") {
setPointer(event, pointer);
raycaster.setFromCamera(pointer, camera);
const point = new THREE.Vector3();
if (raycaster.ray.intersectPlane(dragPlane, point)) {
contentRoot.worldToLocal(point);
transform.translation[0] = point.x;
transform.translation[1] = point.y;
}
} else if (composerTool === "rotate") {
transform.rotation_euler_degrees[2] += (event.clientX - compositionDrag.startX) * 0.45;
} else if (composerTool === "scale") {
const factor = Math.max(0.05, Math.exp((compositionDrag.startY - event.clientY) * 0.01));
transform.scale = compositionDrag.transform.scale.map((value) => Number(value) * factor);
}
compositionDrag.object.userData.compositionTransform = transform;
applyAssetTransform(compositionDrag.object, transform);
selectionHelper?.update();
parent.postMessage({ type: "real2sim-composition-transform", instance_id: compositionDrag.instanceId, transform }, location.origin);
event.preventDefault();
});
const finishDrag = (event) => {
if (!compositionDrag) return;
const instanceId = compositionDrag.instanceId;
compositionDrag = null;
controls.enabled = true;
if (renderer.domElement.hasPointerCapture?.(event.pointerId)) renderer.domElement.releasePointerCapture(event.pointerId);
parent.postMessage({ type: "real2sim-composition-transform-end", instance_id: instanceId }, location.origin);
};
renderer.domElement.addEventListener("pointerup", finishDrag);
renderer.domElement.addEventListener("pointercancel", finishDrag);
function compositionHit(event, raycasterInstance, pointerInstance) {
setPointer(event, pointerInstance);
raycasterInstance.setFromCamera(pointerInstance, camera);
const hits = raycasterInstance.intersectObjects([...draftObjects.values()], true);
for (const result of hits) {
let object = result.object;
while (object && !object.userData.compositionInstanceId) object = object.parent;
if (object?.userData.compositionInstanceId) return object;
}
return null;
}
function setPointer(event, target) {
const rect = renderer.domElement.getBoundingClientRect();
target.x = ((event.clientX - rect.left) / rect.width) * 2 - 1;
target.y = -((event.clientY - rect.top) / rect.height) * 2 + 1;
}
}
async function loadCompositionDraft(payload) {
const requested = [payload.robot, ...(Array.isArray(payload.assets) ? payload.assets : [])].filter(Boolean);
const requestedIds = new Set(requested.map((item) => String(item.instance_id || "")));
for (const [id, object] of draftObjects) {
if (requestedIds.has(id)) continue;
contentRoot.remove(object);
draftObjects.delete(id);
}
for (const asset of requested) {
const instanceId = String(asset.instance_id || "");
if (!instanceId) continue;
let object = draftObjects.get(instanceId);
if (!object) {
object = await createOverlayObject(asset);
if (!object) {
debug.warnings.push(`composition draft ${asset.name || instanceId} has no browser-renderable asset`);
continue;
}
object.userData.compositionInstanceId = instanceId;
object.traverse((child) => { child.userData.compositionInstanceId = instanceId; });
contentRoot.add(object);
draftObjects.set(instanceId, object);
}
object.userData.compositionTransform = cloneTransform(asset.transform || {});
applyAssetTransform(object, object.userData.compositionTransform);
}
selectedDraftId = String(payload.selected_id || selectedDraftId || "");
selectDraftObject(selectedDraftId, false);
debug.compositionDraftCount = draftObjects.size;
debug.compositionObjects = [...draftObjects.entries()].map(([instanceId, object]) => {
const bounds = new THREE.Box3().setFromObject(object);
const size = bounds.getSize(new THREE.Vector3());
const center = bounds.getCenter(new THREE.Vector3());
return { instanceId, size: size.toArray(), center: center.toArray() };
});
document.body.dataset.compositionDraftCount = String(draftObjects.size);
document.body.dataset.compositionBounds = JSON.stringify(debug.compositionObjects);
}
function selectDraftObject(instanceId, notify) {
selectedDraftId = instanceId;
if (selectionHelper) {
scene.remove(selectionHelper);
selectionHelper = null;
}
const object = draftObjects.get(instanceId);
if (object) {
selectionHelper = new THREE.BoxHelper(object, 0x32dcc0);
scene.add(selectionHelper);
}
if (notify) parent.postMessage({ type: "real2sim-composition-selected", instance_id: instanceId }, location.origin);
}
function cloneTransform(transform) {
const translation = Array.isArray(transform.translation) ? transform.translation.map(Number) : [0, 0, 0];
const rotation = Array.isArray(transform.rotation_euler_degrees) ? transform.rotation_euler_degrees.map(Number) : [0, 0, 0];
const scale = Array.isArray(transform.scale) ? transform.scale.map(Number) : [1, 1, 1];
return { translation, rotation_euler_degrees: rotation, scale };
}
function pointSizeFor(count, asset) {
const name = String(asset?.name || asset?.path || "").toLowerCase();
if (name.includes("colmap_sparse") || name.includes("sparse_points")) return 0.022;
@ -764,6 +946,7 @@
function animate() {
controls.update();
selectionHelper?.update();
debug.canvas = { width: renderer.domElement.width, height: renderer.domElement.height };
renderer.render(scene, camera);
requestAnimationFrame(animate);

View File

@ -276,10 +276,62 @@ menu { display: flex; justify-content: flex-end; gap: 8px; margin: 0; padding: 1
.toast.show { opacity: 1; transform: none; }
.viewer-mode [data-open-import], .viewer-mode .bulk-bar button, .viewer-mode [data-delete-dataset], .viewer-mode [data-delete-scene], .viewer-mode [data-delete-model] { display: none; }
.composition-workbench { display: grid; grid-template-columns: minmax(230px, .72fr) minmax(560px, 1.8fr) minmax(270px, .82fr); gap: 10px; align-items: stretch; }
.composition-library, .composition-stage-panel, .composition-inspector { min-height: 640px; }
.composition-library-body, .composition-inspector-body { display: grid; gap: 12px; padding-top: 12px; }
.composition-section-head, .composition-hierarchy-head { display: flex; align-items: center; justify-content: space-between; gap: 10px; color: var(--muted); font-size: 10px; }
.composition-section-head strong, .composition-hierarchy-head strong { color: var(--text); font-size: 12px; }
.composition-resource-list { display: grid; gap: 8px; }
.composition-resource-card { display: grid; grid-template-columns: 54px minmax(0, 1fr); gap: 10px; min-height: 78px; padding: 9px; border: 1px solid var(--line); border-radius: 6px; background: #091720; text-align: left; cursor: grab; }
.composition-resource-card:active { cursor: grabbing; }
.composition-resource-card.active { border-color: var(--accent); box-shadow: inset 0 0 0 1px rgba(52, 209, 187, .16); }
.composition-resource-card .resource-mark { width: 54px; height: 54px; display: grid; place-items: center; overflow: hidden; border: 1px solid var(--line-soft); border-radius: 5px; background: #061016; color: var(--accent); font-size: 10px; font-weight: 750; }
.composition-resource-card .resource-mark img { width: 100%; height: 100%; object-fit: contain; }
.composition-resource-card strong, .composition-asset-card strong { display: block; font-size: 12px; }
.composition-resource-card span, .composition-asset-card span { display: block; margin-top: 4px; color: var(--muted); font-size: 9px; line-height: 1.35; }
.composition-asset-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 7px; max-height: 388px; overflow: auto; padding-right: 2px; }
.composition-asset-card { min-height: 88px; display: grid; align-content: space-between; gap: 8px; padding: 9px; border: 1px solid var(--line); border-radius: 6px; background: #091720; color: var(--text); text-align: left; cursor: grab; }
.composition-asset-card:hover, .composition-resource-card:hover { border-color: #39786f; background: #0b1c25; }
.composition-asset-card[data-in-scene="true"] { border-color: #2b665e; background: #0b211f; }
.composition-asset-card em { color: var(--accent); font-size: 9px; font-style: normal; }
.composition-toolbar { min-height: 48px; display: flex; align-items: center; gap: 5px; padding: 8px 11px; border-bottom: 1px solid var(--line-soft); }
.composition-toolbar button { min-height: 30px; padding: 5px 9px; }
.composition-toolbar button.active { border-color: var(--accent); background: #103b38; color: #eafffb; }
.toolbar-separator { width: 1px; height: 22px; margin: 0 4px; background: var(--line); }
.composition-stage-status { min-width: 0; margin-left: auto; color: var(--muted); font-size: 10px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.composition-canvas-wrap { position: relative; height: 476px; margin: 0 !important; overflow: hidden; background: #030709; }
.composition-canvas-wrap iframe { width: 100%; height: 100%; border: 0; background: #030709; }
.composition-drop-overlay { position: absolute; inset: 10px; z-index: 4; display: none; place-items: center; border: 1px dashed var(--accent); border-radius: 6px; background: rgba(7, 31, 31, .82); color: #d9fff9; font-size: 14px; font-weight: 700; pointer-events: none; }
body.composition-dragging .composition-drop-overlay { display: grid; pointer-events: auto; }
.composition-hierarchy-head { min-height: 34px; padding: 7px 11px; border-top: 1px solid var(--line-soft); border-bottom: 1px solid var(--line-soft); }
.composition-hierarchy { max-height: 120px; overflow: auto; margin: 0 !important; }
.composition-tree-row { min-height: 31px; display: grid; grid-template-columns: minmax(0, 1fr) 92px 42px; align-items: center; gap: 7px; padding: 5px 11px; border-bottom: 1px solid var(--line-soft); cursor: pointer; }
.composition-tree-row:hover, .composition-tree-row.active { background: #0d222b; }
.composition-tree-row.active { box-shadow: inset 2px 0 var(--accent); }
.composition-tree-row span { color: var(--muted); font-size: 9px; }
.composition-tree-row button { min-height: 24px; padding: 2px 6px; font-size: 9px; }
.composition-inspector fieldset { display: grid; gap: 9px; margin: 0; padding: 10px; border: 1px solid var(--line-soft); border-radius: 6px; }
.composition-inspector legend { padding: 0 5px; color: var(--muted); font-size: 10px; }
.composition-selection-card { min-height: 54px; padding: 9px; border: 1px solid var(--line); border-radius: 6px; background: #08141c; }
.composition-selection-card strong { display: block; }
.composition-selection-card span { display: block; margin-top: 4px; color: var(--muted); font-size: 9px; }
.axis-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 7px; }
.axis-grid label { min-width: 0; font-size: 9px; }
.axis-grid input { min-width: 0; padding: 7px 6px; }
.compatibility-card { display: grid; gap: 6px; padding: 10px; border: 1px solid #255f57; border-radius: 6px; background: #09201e; color: var(--muted); font-size: 10px; line-height: 1.45; }
.compatibility-card strong { color: #74dfcb; }
.composition-action-bar { position: sticky; bottom: 8px; z-index: 20; display: grid; grid-template-columns: minmax(240px, 1fr) minmax(180px, .75fr) minmax(180px, .75fr) minmax(180px, .75fr); align-items: end; gap: 9px; margin-top: 10px; padding: 10px 12px; border: 1px solid var(--line); border-radius: 7px; background: rgba(8, 20, 27, .96); box-shadow: 0 14px 40px rgba(0, 0, 0, .38); backdrop-filter: blur(8px); }
.composition-action-bar button { min-height: 39px; }
.composition-tasks-panel { margin-top: 10px; }
.composition-inspector-body > fieldset[disabled] { opacity: .45; }
@media (max-width: 1180px) {
.overview-grid, .two-column, .reconstruct-grid, .scene-workbench { grid-template-columns: 1fr; }
.overview-grid, .two-column, .reconstruct-grid, .scene-workbench, .composition-workbench { grid-template-columns: 1fr; }
.overview-grid .wide { grid-column: auto; }
.scene-workbench .viewer-panel { order: -1; }
.composition-stage-panel { order: -1; }
.composition-library, .composition-stage-panel, .composition-inspector { min-height: 0; }
.composition-action-bar { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.asset-grid, .model-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.selection-card-rail { grid-template-columns: repeat(3, minmax(0, 1fr)); }
}
@ -319,6 +371,12 @@ menu { display: flex; justify-content: flex-end; gap: 8px; margin: 0; padding: 1
.viewer-panel iframe, #scenePreview { width: calc(100% - 26px); min-height: 390px; }
.task-facts { grid-template-columns: 1fr; }
.task-drawer { width: 100vw; }
.composition-canvas-wrap { height: 420px; }
.composition-action-bar { position: static; grid-template-columns: 1fr; }
.composition-asset-grid { display: flex; max-height: none; overflow-x: auto; }
.composition-asset-card { flex: 0 0 140px; }
.composition-toolbar { overflow-x: auto; }
.composition-stage-status { display: none; }
.toolbar { width: 100%; }
.toolbar input, .toolbar select { width: 100%; }
.user-row { align-items: start; flex-direction: column; }

View File

@ -20,6 +20,8 @@ from real2sim_platform.adapters import (
index_artifacts_for_task,
)
from real2sim_platform.motrix_scene import (
COMPOSITION_ASSET_RECIPES,
build_scene_composition,
compose_scene_asset,
convert_asset_to_obj,
detect_spatial_bundle,
@ -206,6 +208,104 @@ def test_scene_revision_copy_is_isolated_and_rebases_package_paths(tmp_path: Pat
assert (source / "scene_manifest.json").read_bytes() == source_manifest_before
def test_scene_composition_catalog_and_payload_are_registered(tmp_path: Path) -> None:
config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api")
app = create_app(config)
package = tmp_path / "editable-scene"
runtime = package / "simulator_assets" / "scene.xml"
runtime.parent.mkdir(parents=True)
runtime.write_text('<mujoco><worldbody><geom type="plane" size="2 2 .1"/></worldbody></mujoco>\n', encoding="utf-8")
manifest = package / "scene_manifest.json"
manifest.write_text(json.dumps({"assets": [], "simulator_artifacts": {"motrixsim": {"runtime_scene": str(runtime)}}}), encoding="utf-8")
scene = PlatformStore(config.db_path).create_scene(
name="editable",
path=runtime,
metadata={"package": str(package), "editable": True},
source_type="motrixsim_spatial_bundle",
manifest_path=manifest,
)
with TestClient(app) as client:
robots = client.get("/api/robots")
assert robots.status_code == 200
assert robots.json()[0]["id"] == "rm65"
assets = client.get("/api/assets").json()
assert {"recipe:coffee_table", "recipe:pick_cube", "recipe:place_target"}.issubset({item["id"] for item in assets})
invalid = client.post(
f"/api/scenes/{scene['id']}/compose",
json={"name": "invalid", "robot": {"robot_id": "rm65"}, "assets": [{"catalog_id": "missing", "instance_id": "missing"}]},
)
assert invalid.status_code == 400
queued = client.post(
f"/api/scenes/{scene['id']}/compose",
json={
"name": "RM65 pick place",
"robot": {"robot_id": "rm65", "transform": {"translation": [1.03, 0, 0.011], "scale": [1, 1, 1]}},
"assets": [
{"catalog_id": recipe_id, "instance_id": f"asset_{index}", "transform": COMPOSITION_ASSET_RECIPES[recipe_id]["default_transform"]}
for index, recipe_id in enumerate(("recipe:coffee_table", "recipe:pick_cube", "recipe:place_target"))
],
"validation_steps": 120,
},
)
assert queued.status_code == 200, queued.text
assert queued.json()["task"]["type"] == "scene_composition"
@pytest.mark.integration
def test_real_rm65_scene_composition_loads_and_steps_in_both_runtimes(tmp_path: Path) -> None:
mujoco = pytest.importorskip("mujoco")
pytest.importorskip("motrixsim")
candidates = [
Path("/opt/motrixsim-docs/examples/assets/rm65_open_cabinet/mjx_rm65.xml"),
Path("/Users/d-robotics/workSpace/motrixsim-docs/examples/assets/rm65_open_cabinet/mjx_rm65.xml"),
]
robot_source = next((path for path in candidates if path.is_file()), None)
if robot_source is None:
pytest.skip("real MotrixSim RM65 asset checkout is unavailable")
package = tmp_path / "composition"
runtime = package / "simulator_assets" / "scene.xml"
(package / "assets" / "meshes").mkdir(parents=True)
runtime.parent.mkdir(parents=True)
runtime.write_text(
'<mujoco model="composition"><compiler angle="degree" meshdir="../assets/meshes" autolimits="true"/><option timestep="0.002"/><worldbody><geom type="plane" size="3 3 .1"/></worldbody></mujoco>\n',
encoding="utf-8",
)
(package / "scene_manifest.json").write_text(json.dumps({"assets": [], "simulator_artifacts": {"motrixsim": {"runtime_scene": str(runtime)}}}), encoding="utf-8")
(package / "motrix_import.json").write_text(json.dumps({"runtime_scene": str(runtime)}), encoding="utf-8")
asset_specs = [
{
"catalog_id": recipe_id,
"instance_id": f"asset_{index}",
"name": COMPOSITION_ASSET_RECIPES[recipe_id]["name"],
"transform": COMPOSITION_ASSET_RECIPES[recipe_id]["default_transform"],
}
for index, recipe_id in enumerate(("recipe:coffee_table", "recipe:pick_cube", "recipe:place_target"))
]
result = build_scene_composition(
package,
robot_source=robot_source,
robot_spec={"robot_id": "rm65", "transform": {"translation": [1.03, 0, 0.011], "rotation_euler_degrees": [0, 0, 0], "scale": [1, 1, 1]}},
asset_specs=asset_specs,
source_paths={},
validation_steps=120,
)
assert result["validation"]["mujoco"]["status"] == "loaded_and_stepped"
assert result["validation"]["motrixsim"]["status"] == "loaded_and_stepped"
assert result["collection_ready"] is True
model = mujoco.MjModel.from_xml_path(str(runtime))
assert model.nkey >= 1
for object_type, name in (
(mujoco.mjtObj.mjOBJ_BODY, "base_link"),
(mujoco.mjtObj.mjOBJ_BODY, "pick_cube"),
(mujoco.mjtObj.mjOBJ_BODY, "place_target"),
(mujoco.mjtObj.mjOBJ_JOINT, "pick_cube_free"),
(mujoco.mjtObj.mjOBJ_GEOM, "pick_cube_geom"),
(mujoco.mjtObj.mjOBJ_CAMERA, "overview"),
):
assert mujoco.mj_name2id(model, object_type, name) >= 0
assert Path(result["composition"]["robot"]["path"]).stat().st_size > 1_000_000
def test_ascii_3dgs_spz_exporter_uses_real_spark_writer(tmp_path: Path) -> None:
config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace")
probe = _spz_exporter_probe(config)