diff --git a/deploy/poc/app-template.yaml b/deploy/poc/app-template.yaml index 5e859ea..9335c35 100644 --- a/deploy/poc/app-template.yaml +++ b/deploy/poc/app-template.yaml @@ -26,6 +26,7 @@ data: anysplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python" cuda_home = "/usr/local/cuda" runtime_env_script = "/opt/gs_playground/scripts/real2sim_wuying_paper_env.sh" + motrixsim_docs_root = "/opt/motrixsim-docs" allowed_path_roots = "/var/lib/real2sim-poc:/opt/DISCOVERSE:/opt/gs_playground:/opt/motrixsim-docs:/opt/real2sim_paper_runtime:/tmp" production_mode = true auth_enabled = true diff --git a/docs/prototypes/scene-composition-v1.png b/docs/prototypes/scene-composition-v1.png new file mode 100644 index 0000000..e1126ff Binary files /dev/null and b/docs/prototypes/scene-composition-v1.png differ diff --git a/pyproject.toml b/pyproject.toml index 04b16cd..4e7c333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ where = ["src"] "static/vendor/three/addons/loaders/*", "static/vendor/three/addons/postprocessing/*", "static/vendor/three/addons/utils/*", + "static/assets/robots/*", ] [tool.pytest.ini_options] diff --git a/scripts/export_rm65_browser_asset.py b/scripts/export_rm65_browser_asset.py new file mode 100644 index 0000000..c2ac849 --- /dev/null +++ b/scripts/export_rm65_browser_asset.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from real2sim_platform.motrix_scene import export_mujoco_body_glb + + +def main() -> int: + parser = argparse.ArgumentParser(description="Export the registered RM65 MJCF visual geometry as a browser GLB") + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = export_mujoco_body_glb(args.source.resolve(), args.output.resolve(), root_body="base_link", relative_to_root=True) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/real2sim_platform/adapters.py b/src/real2sim_platform/adapters.py index b4b2d87..2ba3613 100644 --- a/src/real2sim_platform/adapters.py +++ b/src/real2sim_platform/adapters.py @@ -17,7 +17,7 @@ from urllib.parse import quote, unquote, urlparse from urllib.request import urlopen from .config import PlatformConfig, is_path_allowed -from .motrix_scene import build_spatial_scene_package, compose_scene_asset +from .motrix_scene import COMPOSITION_ASSET_RECIPES, build_scene_composition, build_spatial_scene_package, compose_scene_asset from .source_revision import read_source_revision from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path from .store import PlatformStore @@ -194,6 +194,8 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A return run_motrixsim_scene_import(config, store, task, emit) if task_type == "scene_asset_compose": return run_scene_asset_compose(config, store, task, emit) + if task_type == "scene_composition": + return run_scene_composition(config, store, task, emit) if task_type == "simulator_validation": return run_simulator_validation(config, store, task, emit) if task_type == "expert_collection": @@ -881,6 +883,104 @@ def run_scene_asset_compose(config: PlatformConfig, store: PlatformStore, task: } +def run_scene_composition(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: + params = task["params"] + parent = store.get_scene(str(params["scene_id"])) + source_package = Path(str((parent.get("metadata") or {}).get("package") or "")) + if not source_package.is_dir(): + scene_path = Path(parent["path"]) + source_package = scene_path if scene_path.is_dir() else scene_path.parent + if not (source_package / "scene_manifest.json").is_file(): + raise ValueError("selected scene is not an editable MotrixSim scene package") + + run_dir = Path(task["run_dir"]) + composition_dir = run_dir / "scene_composition" + _task_progress(store, task, 12, "创建组合版本", "正在复制基础场景,原场景和原始重建产物保持不变") + _copy_tree_isolated(source_package, composition_dir) + _rebase_package_metadata(composition_dir, source_package) + + composition = params.get("composition") or {} + asset_specs = list(composition.get("assets") or []) + source_paths: dict[str, Path] = {} + for index, spec in enumerate(asset_specs): + catalog_id = str(spec.get("catalog_id") or "") + if catalog_id in COMPOSITION_ASSET_RECIPES or catalog_id.startswith("primitive:"): + continue + instance_id = str(spec.get("instance_id") or f"asset_{index + 1}") + source_paths[instance_id] = _resolve_asset_source(config, store, spec.get("source") or {}, spec.get("path")) + + robot_source = config.motrixsim_docs_root / "examples" / "assets" / "rm65_open_cabinet" / "mjx_rm65.xml" + _task_progress(store, task, 28, "合并 RM65", "正在写入真实关节、执行器、夹爪碰撞体和 MotrixSim 资产") + result = build_scene_composition( + composition_dir, + robot_source=robot_source, + robot_spec=composition.get("robot") or {}, + asset_specs=asset_specs, + source_paths=source_paths, + validation_steps=int(params.get("validation_steps", 120)), + emit=emit, + ) + + _task_progress(store, task, 82, "登记组合场景", "双运行时校验通过,正在登记预览资产、控制器和场景血缘") + artifacts = _index_artifacts(store, task["id"], composition_dir) + manifest_path = Path(result["manifest_path"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + revision_id = uuid.uuid4().hex + manifest["revision"] = { + "id": revision_id, + "parent_scene_id": parent["id"], + "reason": "motrixsim_scene_composition", + "task_id": task["id"], + } + _attach_artifact_urls(manifest, artifacts) + manifest_path.write_text(json.dumps(enrich_scene_manifest_assets(manifest), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + validation = result["validation"] + metadata = dict(parent.get("metadata") or {}) + metadata.update( + { + "package": str(composition_dir), + "parent_scene_id": parent["id"], + "revision_reason": "motrixsim_scene_composition", + "runtime_validation": {"mujoco": bool(validation["mujoco"]["ok"]), "motrixsim": bool(validation["motrixsim"]["ok"])}, + "editable": True, + "robot_id": "rm65", + "controller": "rm65_pick_place_6d_ik" if result["collection_ready"] else None, + "collection_ready": bool(result["collection_ready"]), + } + ) + name = str(params.get("name") or composition.get("name") or f"{parent['name']} · RM65 组合") + scene = store.create_scene( + item_id=revision_id, + name=name, + path=Path(result["runtime_scene"]), + metadata=metadata, + source_type="scene_composition", + source_uri=parent.get("storage_uri") or parent.get("source_uri") or parent["path"], + version=_next_scene_revision(str(parent.get("version") or "v1")), + parent_task_id=task["id"], + manifest_path=manifest_path, + metrics={ + **(parent.get("metrics") or {}), + "composition_asset_count": len(asset_specs), + "robot_count": 1, + "collection_ready": bool(result["collection_ready"]), + }, + tags=list(dict.fromkeys([*(parent.get("tags") or []), "scene-composition", "motrixsim", "rm65", "runtime-validated", *( ["collection-ready"] if result["collection_ready"] else [])])), + cache_state="local", + ) + _task_progress(store, task, 96, "组合场景可用", "组合场景已可测试;满足控制器对象约定时可直接进入专家采集") + return { + "scene_id": scene["id"], + "parent_scene_id": parent["id"], + "scene_path": scene["path"], + "manifest_path": str(manifest_path), + "composition": result["composition"], + "collection_ready": bool(result["collection_ready"]), + "runtime_validation": validation, + "report_path": result["report_path"], + } + + def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: params = task["params"] run_dir = Path(task["run_dir"]) diff --git a/src/real2sim_platform/config.py b/src/real2sim_platform/config.py index e2d387c..db0a63b 100644 --- a/src/real2sim_platform/config.py +++ b/src/real2sim_platform/config.py @@ -32,6 +32,7 @@ class PlatformConfig: anysplat_python: Path = Path("/opt/real2sim_paper_runtime/envs/anysplat/bin/python") cuda_home: Path = Path("/usr/local/cuda-12.1") runtime_env_script: Path = Path("/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh") + motrixsim_docs_root: Path = Path("/opt/motrixsim-docs") allowed_path_roots: tuple[Path, ...] = () worker_mode: str = "embedded" worker_id: str = "local-worker" @@ -99,6 +100,7 @@ class PlatformConfig: self.discoverse_root, self.runtime_root, self.anysplat_root, + self.motrixsim_docs_root, Path("/opt/gs_playground_real2sim_48q"), Path("/tmp"), ) @@ -129,6 +131,7 @@ def load_config(path: Path | None = None) -> PlatformConfig: anysplat_python=Path(str(values.get("anysplat_python", values.get("gsplat_python", "/opt/real2sim_paper_runtime/envs/anysplat/bin/python")))), cuda_home=Path(str(values.get("cuda_home", "/usr/local/cuda-12.1"))), runtime_env_script=Path(str(values.get("runtime_env_script", "/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh"))), + motrixsim_docs_root=Path(str(values.get("motrixsim_docs_root", os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_ROOT", "/opt/motrixsim-docs")))), allowed_path_roots=_parse_path_tuple(str(values.get("allowed_path_roots", ""))), worker_mode=str(values.get("worker_mode", os.environ.get("REAL2SIM_WORKER_MODE", "embedded"))), worker_id=str(values.get("worker_id", os.environ.get("REAL2SIM_WORKER_ID", "local-worker"))), @@ -179,6 +182,7 @@ def write_default_config(path: Path, config: PlatformConfig | None = None) -> No f'anysplat_python = "{cfg.anysplat_python.as_posix()}"', f'cuda_home = "{cfg.cuda_home.as_posix()}"', f'runtime_env_script = "{cfg.runtime_env_script.as_posix()}"', + f'motrixsim_docs_root = "{cfg.motrixsim_docs_root.as_posix()}"', f'allowed_path_roots = "{_format_path_tuple(cfg.effective_allowed_path_roots)}"', f'worker_mode = "{cfg.worker_mode}"', f'worker_id = "{cfg.worker_id}"', diff --git a/src/real2sim_platform/motrix_scene.py b/src/real2sim_platform/motrix_scene.py index d5e33a6..5b23f19 100644 --- a/src/real2sim_platform/motrix_scene.py +++ b/src/real2sim_platform/motrix_scene.py @@ -6,6 +6,7 @@ import math import os import shutil import time +from copy import deepcopy import xml.etree.ElementTree as ET import zipfile from pathlib import Path, PurePosixPath @@ -17,6 +18,66 @@ SCENE_MANIFEST_SCHEMA = "real2sim.scene_manifest.v3" MOTRIX_IMPORT_SCHEMA = "real2sim.motrixsim_scene.v2" SUPPORTED_MESH_SUFFIXES = {".obj", ".stl", ".ply", ".glb", ".gltf", ".usd", ".usda", ".usdc", ".usdz"} SUPPORTED_PRIMITIVES = {"box", "sphere", "cylinder", "capsule"} +SCENE_COMPOSITION_SCHEMA = "real2sim.scene_composition.v1" + +COMPOSITION_ASSET_RECIPES: dict[str, dict[str, Any]] = { + "recipe:coffee_table": { + "id": "recipe:coffee_table", + "name": "茶几", + "description": "静态碰撞茶几,适用于桌面操作任务", + "runtime_name": "coffee_table", + "category": "task_fixture", + "default_transform": {"translation": [0.65, 0.0, 0.25], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "physics": {"dynamic": False, "mass": 8.0, "collision_mode": "visual_and_collision", "dimensions": [0.9, 0.64, 0.34], "friction": [0.8, 0.01, 0.001], "rgba": [0.45, 0.30, 0.18, 1.0]}, + "components": [ + {"name": "top", "primitive": "box", "position": [0.0, 0.0, 0.0], "dimensions": [0.9, 0.64, 0.06], "rgba": [0.45, 0.30, 0.18, 1.0]}, + {"name": "leg_1", "primitive": "box", "position": [0.36, 0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]}, + {"name": "leg_2", "primitive": "box", "position": [-0.36, 0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]}, + {"name": "leg_3", "primitive": "box", "position": [0.36, -0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]}, + {"name": "leg_4", "primitive": "box", "position": [-0.36, -0.25, -0.14], "dimensions": [0.05, 0.05, 0.28], "rgba": [0.20, 0.13, 0.08, 1.0]}, + ], + "required_for_collection": True, + }, + "recipe:pick_cube": { + "id": "recipe:pick_cube", + "name": "抓取方块", + "description": "带自由关节和真实接触的动态抓取目标", + "runtime_name": "pick_cube", + "category": "task_object", + "default_transform": {"translation": [0.65, 0.0, 0.33], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "physics": {"dynamic": True, "mass": 0.05, "collision_mode": "visual_and_collision", "dimensions": [0.06, 0.06, 0.06], "friction": [0.9, 0.01, 0.001], "rgba": [0.9, 0.2, 0.12, 1.0]}, + "components": [{"name": "geom", "primitive": "box", "position": [0.0, 0.0, 0.0], "dimensions": [0.06, 0.06, 0.06], "rgba": [0.9, 0.2, 0.12, 1.0]}], + "required_for_collection": True, + }, + "recipe:place_target": { + "id": "recipe:place_target", + "name": "放置目标", + "description": "不参与碰撞的半透明任务目标区域", + "runtime_name": "place_target", + "category": "task_marker", + "default_transform": {"translation": [0.45, 0.18, 0.335], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "physics": {"dynamic": False, "mass": 0.01, "collision_mode": "visual_only", "dimensions": [0.11, 0.11, 0.008], "friction": [0.0, 0.0, 0.0], "rgba": [0.1, 0.8, 0.6, 0.35]}, + "components": [{"name": "marker", "primitive": "cylinder", "position": [0.0, 0.0, 0.0], "dimensions": [0.11, 0.11, 0.008], "rgba": [0.1, 0.8, 0.6, 0.35], "collision_mode": "visual_only"}], + "required_for_collection": True, + }, + "recipe:storage_bin": { + "id": "recipe:storage_bin", + "name": "收纳盒", + "description": "由底板和四面碰撞壁组成的静态容器", + "runtime_name": "storage_bin", + "category": "prop", + "default_transform": {"translation": [0.45, -0.22, 0.37], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "physics": {"dynamic": False, "mass": 0.6, "collision_mode": "visual_and_collision", "dimensions": [0.28, 0.20, 0.12], "friction": [0.75, 0.01, 0.001], "rgba": [0.72, 0.72, 0.64, 1.0]}, + "components": [ + {"name": "base", "primitive": "box", "position": [0.0, 0.0, -0.05], "dimensions": [0.28, 0.20, 0.02]}, + {"name": "wall_front", "primitive": "box", "position": [0.0, -0.095, 0.0], "dimensions": [0.28, 0.01, 0.12]}, + {"name": "wall_back", "primitive": "box", "position": [0.0, 0.095, 0.0], "dimensions": [0.28, 0.01, 0.12]}, + {"name": "wall_left", "primitive": "box", "position": [-0.135, 0.0, 0.0], "dimensions": [0.01, 0.18, 0.12]}, + {"name": "wall_right", "primitive": "box", "position": [0.135, 0.0, 0.0], "dimensions": [0.01, 0.18, 0.12]}, + ], + "required_for_collection": False, + }, +} def detect_spatial_bundle(root: Path) -> dict[str, Any]: @@ -243,13 +304,18 @@ def compose_scene_asset( if worldbody is None: worldbody = ET.SubElement(root, "worldbody") + compiler = root.find("compiler") + compiler_angle = str(compiler.attrib.get("angle", "degree") if compiler is not None else "degree").lower() + euler = transform["rotation_euler_degrees"] + if compiler_angle.startswith("rad"): + euler = [math.radians(value) for value in euler] body = ET.SubElement( worldbody, "body", { "name": asset_key, "pos": _numbers(transform["translation"]), - "euler": _numbers(transform["rotation_euler_degrees"]), + "euler": _numbers(euler), }, ) if physics["dynamic"]: @@ -348,6 +414,556 @@ def compose_scene_asset( } +def composition_asset_catalog() -> list[dict[str, Any]]: + return [deepcopy(COMPOSITION_ASSET_RECIPES[key]) for key in sorted(COMPOSITION_ASSET_RECIPES)] + + +def build_scene_composition( + package_dir: Path, + *, + robot_source: Path, + robot_spec: dict[str, Any], + asset_specs: list[dict[str, Any]], + source_paths: dict[str, Path] | None, + validation_steps: int, + emit: Callable[[str], None] | None = None, +) -> dict[str, Any]: + emit = emit or (lambda _message: None) + source_paths = source_paths or {} + manifest_path = package_dir / "scene_manifest.json" + manifest = _read_json(manifest_path) + runtime_scene = _runtime_scene_from_manifest(package_dir, manifest) + if not runtime_scene.is_file(): + raise FileNotFoundError(f"MotrixSim runtime scene not found: {runtime_scene}") + if not robot_source.is_file(): + raise FileNotFoundError(f"RM65 source MJCF not found: {robot_source}") + + tree = ET.parse(runtime_scene) + root = tree.getroot() + if root.tag != "mujoco": + raise ValueError("scene runtime file is not an MJCF document") + compiler = _ensure_radian_compiler(root) + emit("Merging the real RM65 articulated model into the MotrixSim scene") + robot_entry = _merge_rm65_robot( + root, + runtime_scene=runtime_scene, + compiler=compiler, + robot_source=robot_source, + robot_spec=robot_spec, + ) + + entries: list[dict[str, Any]] = [] + seen_instances: set[str] = set() + seen_runtime_names: set[str] = set() + for index, spec in enumerate(asset_specs): + instance_id = _safe_name(str(spec.get("instance_id") or f"asset_{index + 1}")) + if instance_id in seen_instances: + raise ValueError(f"duplicate composition asset instance_id: {instance_id}") + seen_instances.add(instance_id) + catalog_id = str(spec.get("catalog_id") or "").strip() + if catalog_id in COMPOSITION_ASSET_RECIPES or catalog_id.startswith("primitive:"): + entry = _append_composition_primitive(root, spec=spec, instance_id=instance_id, seen_runtime_names=seen_runtime_names) + else: + source = source_paths.get(instance_id) + if source is None: + raise ValueError(f"composition asset is not registered or has no source file: {catalog_id or instance_id}") + entry = _append_composition_mesh( + root, + package_dir=package_dir, + runtime_scene=runtime_scene, + compiler=compiler, + spec=spec, + instance_id=instance_id, + source_path=source, + seen_runtime_names=seen_runtime_names, + emit=emit, + ) + entries.append(entry) + + runtime_names = {str(entry.get("runtime", {}).get("body") or "") for entry in entries} + collection_ready = {"coffee_table", "pick_cube", "place_target"}.issubset(runtime_names) + _ensure_overview_camera(root, target="pick_cube" if "pick_cube" in runtime_names else "base_link") + _indent_xml(root) + tree.write(runtime_scene, encoding="utf-8", xml_declaration=True) + _write_home_keyframe(runtime_scene) + + robot_glb = package_dir / "viewer" / "robots" / "rm65.glb" + emit("Exporting the articulated RM65 visual geometry for the browser editor") + robot_preview = export_mujoco_body_glb(runtime_scene, robot_glb, root_body="base_link") + robot_entry.update( + { + "path": str(robot_glb), + "size_bytes": robot_glb.stat().st_size, + "sha256": _sha256(robot_glb), + "preview": robot_preview, + } + ) + + emit("Validating the composed scene in MuJoCo and MotrixSim") + validation = validate_runtime_scene(runtime_scene, steps=validation_steps) + reports_dir = package_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + validation_path = reports_dir / "scene_composition_validation.json" + _write_json(validation_path, validation) + if not validation["ok"]: + failed = [name for name in ("mujoco", "motrixsim") if not validation[name]["ok"]] + raise RuntimeError("Composed scene failed runtime validation: " + "; ".join(f"{name}: {validation[name].get('error', 'unknown error')}" for name in failed)) + + for entry in [robot_entry, *entries]: + entry.setdefault("runtime", {})["validated"] = True + composition = { + "schema": SCENE_COMPOSITION_SCHEMA, + "simulator": "motrixsim", + "robot": robot_entry, + "assets": entries, + "controller": "rm65_pick_place_6d_ik" if collection_ready else None, + "collection_ready": collection_ready, + "required_collection_objects": ["coffee_table", "pick_cube", "place_target"], + "validation_steps": validation_steps, + "updated_at": _utc_now(), + } + report_path = reports_dir / "scene_composition_report.json" + _write_json( + report_path, + { + "schema": "real2sim.scene_composition_report.v1", + "ok": True, + "runtime_scene": str(runtime_scene), + "robot_source": str(robot_source), + "robot_body_count": robot_preview["body_count"], + "robot_mesh_count": robot_preview["mesh_count"], + "asset_count": len(entries), + "collection_ready": collection_ready, + "runtime_validation": validation, + }, + ) + existing_assets = [ + item + for item in manifest.get("assets", []) + if not (isinstance(item, dict) and item.get("role") in {"robot", "added_asset"}) + ] + manifest["assets"] = [robot_entry, *entries, *existing_assets] + manifest["composition"] = composition + manifest["updated_at"] = _utc_now() + manifest.setdefault("asset_editing", {}).update({"enabled": True, "revisioned": True, "composition": True}) + manifest.setdefault("simulator_artifacts", {}).setdefault("motrixsim", {}).update({"runtime_scene": str(runtime_scene), "validated": True}) + manifest.setdefault("simulator_artifacts", {}).setdefault("mujoco", {}).update({"mjcf": str(runtime_scene), "validated": True}) + _write_json(manifest_path, manifest) + + descriptor_path = package_dir / "motrix_import.json" + descriptor = _read_json(descriptor_path) if descriptor_path.is_file() else {"schema": MOTRIX_IMPORT_SCHEMA} + descriptor.update({"runtime_scene": str(runtime_scene), "composition": composition, "runtime_validation": validation, "updated_at": _utc_now()}) + _write_json(descriptor_path, descriptor) + return { + "manifest_path": str(manifest_path), + "runtime_scene": str(runtime_scene), + "validation_path": str(validation_path), + "report_path": str(report_path), + "validation": validation, + "composition": composition, + "collection_ready": collection_ready, + } + + +def export_mujoco_body_glb(scene_path: Path, output: Path, *, root_body: str, relative_to_root: bool = False) -> dict[str, Any]: + try: + import mujoco + import numpy as np + import trimesh + except ImportError as exc: + raise RuntimeError("RM65 GLB export requires mujoco, numpy and trimesh") from exc + + model = mujoco.MjModel.from_xml_path(str(scene_path)) + data = mujoco.MjData(model) + if model.nkey: + mujoco.mj_resetDataKeyframe(model, data, 0) + mujoco.mj_forward(model, data) + root_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, root_body) + if root_id < 0: + raise ValueError(f"robot root body is missing: {root_body}") + root_position = np.asarray(data.xpos[root_id], dtype=np.float64) + root_rotation = np.asarray(data.xmat[root_id], dtype=np.float64).reshape(3, 3) + + def is_descendant(body_id: int) -> bool: + current = body_id + while current > 0: + if current == root_id: + return True + current = int(model.body_parentid[current]) + return current == root_id + + scene = trimesh.Scene() + mesh_count = 0 + body_ids: set[int] = set() + for geom_id in range(model.ngeom): + body_id = int(model.geom_bodyid[geom_id]) + mesh_id = int(model.geom_dataid[geom_id]) + if not is_descendant(body_id) or int(model.geom_type[geom_id]) != int(mujoco.mjtGeom.mjGEOM_MESH) or mesh_id < 0: + continue + vertex_start = int(model.mesh_vertadr[mesh_id]) + vertex_count = int(model.mesh_vertnum[mesh_id]) + face_start = int(model.mesh_faceadr[mesh_id]) + face_count = int(model.mesh_facenum[mesh_id]) + vertices = np.asarray(model.mesh_vert[vertex_start : vertex_start + vertex_count], dtype=np.float64).copy() + faces = np.asarray(model.mesh_face[face_start : face_start + face_count], dtype=np.int64).copy() + rotation = np.asarray(data.geom_xmat[geom_id], dtype=np.float64).reshape(3, 3) + vertices = vertices @ rotation.T + np.asarray(data.geom_xpos[geom_id], dtype=np.float64) + if relative_to_root: + vertices = (vertices - root_position) @ root_rotation + mesh = trimesh.Trimesh(vertices=vertices, faces=faces, process=False, validate=False) + rgba = np.clip(np.asarray(model.geom_rgba[geom_id]) * 255, 0, 255).astype(np.uint8) + mesh.visual.vertex_colors = np.tile(rgba, (len(mesh.vertices), 1)) + name = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, geom_id) or f"rm65_geom_{geom_id}" + scene.add_geometry(mesh, geom_name=name, node_name=name) + mesh_count += 1 + body_ids.add(body_id) + if not mesh_count: + raise ValueError("RM65 model contains no exportable visual mesh geometry") + output.parent.mkdir(parents=True, exist_ok=True) + scene.export(output, file_type="glb") + return {"path": str(output), "mesh_count": mesh_count, "body_count": len(body_ids), "size_bytes": output.stat().st_size} + + +def _ensure_radian_compiler(root: ET.Element) -> ET.Element: + compiler = root.find("compiler") + if compiler is None: + compiler = ET.Element("compiler", {"angle": "radian", "autolimits": "true"}) + root.insert(0, compiler) + return compiler + if not str(compiler.attrib.get("angle", "degree")).lower().startswith("rad"): + for element in root.iter(): + if "euler" not in element.attrib: + continue + values = [float(value) for value in element.attrib["euler"].split()] + element.attrib["euler"] = _numbers([math.radians(value) for value in values]) + compiler.attrib["angle"] = "radian" + compiler.attrib.setdefault("autolimits", "true") + return compiler + + +def _merge_rm65_robot( + root: ET.Element, + *, + runtime_scene: Path, + compiler: ET.Element, + robot_source: Path, + robot_spec: dict[str, Any], +) -> dict[str, Any]: + if str(robot_spec.get("robot_id") or "") != "rm65": + raise ValueError("only the registered RM65 robot can be composed in this release") + if root.find("./worldbody/body[@name='base_link']") is not None: + raise ValueError("the scene already contains an RM65 base_link body") + transform = normalize_transform(robot_spec.get("transform") or {}) + if any(abs(value - 1.0) > 1e-9 for value in transform["scale"]): + raise ValueError("articulated RM65 scale must remain [1, 1, 1] to preserve joint and inertia calibration") + + robot_root = ET.parse(robot_source).getroot() + robot_compiler = robot_root.find("compiler") + asset_dir = (robot_source.parent / str(robot_compiler.attrib.get("assetdir", "assets") if robot_compiler is not None else "assets")).resolve() + if not asset_dir.is_dir(): + raise FileNotFoundError(f"RM65 asset directory is missing: {asset_dir}") + mesh_dir = (runtime_scene.parent / str(compiler.attrib.get("meshdir") or ".")).resolve() + robot_mesh_dir = mesh_dir / "robots" / "rm65" + if robot_mesh_dir.exists(): + shutil.rmtree(robot_mesh_dir) + shutil.copytree(asset_dir, robot_mesh_dir) + license_source = robot_source.parent / "LICENSE" + if license_source.is_file(): + shutil.copy2(license_source, robot_mesh_dir / "LICENSE") + + robot_default = robot_root.find("default") + if robot_default is not None: + target_default = root.find("default") + if target_default is None: + target_default = ET.Element("default") + _insert_before(root, target_default, {"asset", "worldbody"}) + for child in robot_default: + target_default.append(deepcopy(child)) + + target_asset = root.find("asset") + if target_asset is None: + target_asset = ET.Element("asset") + _insert_before(root, target_asset, {"worldbody"}) + source_asset = robot_root.find("asset") + if source_asset is not None: + for child in source_asset: + copied = deepcopy(child) + if copied.get("file"): + copied.set("file", f"robots/rm65/{copied.get('file')}") + target_asset.append(copied) + + worldbody = root.find("worldbody") + if worldbody is None: + worldbody = ET.SubElement(root, "worldbody") + robot_body = robot_root.find("./worldbody/body[@name='base_link']") + if robot_body is None: + raise ValueError("RM65 source MJCF does not contain base_link") + copied_body = deepcopy(robot_body) + copied_body.set("pos", _numbers(transform["translation"])) + copied_body.set("euler", _numbers([math.radians(value) for value in transform["rotation_euler_degrees"]])) + worldbody.append(copied_body) + + for section_name in ("equality", "actuator", "contact"): + source_section = robot_root.find(section_name) + if source_section is None: + continue + target_section = root.find(section_name) + if target_section is None: + target_section = ET.SubElement(root, section_name) + for child in source_section: + target_section.append(deepcopy(child)) + return { + "name": "RM65 双指夹爪", + "kind": "robot", + "role": "robot", + "format": "glb", + "autoload": True, + "viewer_hint": "overlay", + "transform": {"translation": [0.0, 0.0, 0.0], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "source_transform": transform, + "robot_id": "rm65", + "controller": "rm65_pick_place_6d_ik", + "runtime": {"body": "base_link", "scene": str(runtime_scene), "validated": False}, + "source": {"path": str(robot_source), "repository": "Motphys/motrixsim-docs"}, + "added_at": _utc_now(), + } + + +def _append_composition_primitive( + root: ET.Element, + *, + spec: dict[str, Any], + instance_id: str, + seen_runtime_names: set[str], +) -> dict[str, Any]: + catalog_id = str(spec.get("catalog_id") or "") + if catalog_id.startswith("primitive:"): + primitive = catalog_id.split(":", 1)[1] + if primitive not in SUPPORTED_PRIMITIVES: + raise ValueError(f"unsupported composition primitive: {primitive}") + recipe = { + "id": catalog_id, + "name": str(spec.get("name") or primitive), + "runtime_name": instance_id, + "default_transform": {"translation": [0.0, 0.0, 0.1], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "physics": {"dynamic": False, "mass": 0.25, "collision_mode": "visual_and_collision", "dimensions": [0.1, 0.1, 0.1], "friction": [0.8, 0.01, 0.001], "rgba": [0.13, 0.76, 0.67, 1.0]}, + "components": [{"name": "geom", "primitive": primitive, "position": [0.0, 0.0, 0.0], "dimensions": [0.1, 0.1, 0.1]}], + } + else: + recipe = COMPOSITION_ASSET_RECIPES[catalog_id] + runtime_name = str(recipe["runtime_name"]) + if runtime_name in seen_runtime_names or root.find(f"./worldbody/body[@name='{runtime_name}']") is not None: + raise ValueError(f"composition already contains runtime object: {runtime_name}") + seen_runtime_names.add(runtime_name) + transform = normalize_transform(spec.get("transform") or recipe.get("default_transform") or {}) + physics = normalize_physics({**(recipe.get("physics") or {}), **(spec.get("physics") or {})}) + worldbody = root.find("worldbody") + if worldbody is None: + worldbody = ET.SubElement(root, "worldbody") + body = ET.SubElement( + worldbody, + "body", + { + "name": runtime_name, + "pos": _numbers(transform["translation"]), + "euler": _numbers([math.radians(value) for value in transform["rotation_euler_degrees"]]), + }, + ) + if physics["dynamic"]: + joint_name = "pick_cube_free" if runtime_name == "pick_cube" else f"{runtime_name}_free" + ET.SubElement(body, "freejoint", {"name": joint_name}) + + viewer_components: list[dict[str, Any]] = [] + collidable = [component for component in recipe["components"] if component.get("collision_mode", physics["collision_mode"]) != "visual_only"] + mass_per_geom = physics["mass"] / max(1, len(collidable)) + for component in recipe["components"]: + primitive = str(component["primitive"]) + dimensions = [float(value) for value in component.get("dimensions") or physics["dimensions"]] + scaled_dimensions = [value * transform["scale"][index] for index, value in enumerate(dimensions)] + local_position = [float(value) for value in component.get("position") or [0.0, 0.0, 0.0]] + scaled_position = [value * transform["scale"][index] for index, value in enumerate(local_position)] + collision_mode = str(component.get("collision_mode") or physics["collision_mode"]) + rgba = [float(value) for value in component.get("rgba") or physics["rgba"]] + component_name = str(component.get("name") or "geom") + if runtime_name == "pick_cube": + geom_name = "pick_cube_geom" + elif runtime_name == "place_target": + geom_name = "place_target_marker" + elif runtime_name == "coffee_table" and component_name == "top": + geom_name = "coffee_table_top" + else: + geom_name = f"{runtime_name}_{_safe_name(component_name)}" + attributes = { + "name": geom_name, + "pos": _numbers(scaled_position), + "friction": _numbers(physics["friction"]), + "contype": "0" if collision_mode == "visual_only" else "1", + "conaffinity": "0" if collision_mode == "visual_only" else "1", + "group": "2" if collision_mode == "visual_only" else "3", + "rgba": _numbers(rgba), + **_primitive_geom(primitive, scaled_dimensions), + } + if physics["dynamic"] and collision_mode != "visual_only": + attributes["mass"] = f"{mass_per_geom:.9g}" + ET.SubElement(body, "geom", attributes) + viewer_components.append( + { + "name": component_name, + "viewer_primitive": primitive, + "translation": local_position, + "rotation_euler_degrees": component.get("rotation_euler_degrees") or [0.0, 0.0, 0.0], + "dimensions": dimensions, + "rgba": rgba, + } + ) + return { + "name": str(spec.get("name") or recipe["name"]), + "path": "", + "kind": "primitive" if len(viewer_components) == 1 else "compound_primitive", + "role": "added_asset", + "format": str(viewer_components[0]["viewer_primitive"]) if len(viewer_components) == 1 else "compound", + "autoload": True, + "viewer_hint": "overlay", + "viewer_primitive": viewer_components[0]["viewer_primitive"] if len(viewer_components) == 1 else None, + "viewer_components": viewer_components, + "catalog_id": catalog_id, + "instance_id": instance_id, + "transform": transform, + "physics": physics, + "runtime": {"body": runtime_name, "validated": False}, + "added_at": _utc_now(), + } + + +def _append_composition_mesh( + root: ET.Element, + *, + package_dir: Path, + runtime_scene: Path, + compiler: ET.Element, + spec: dict[str, Any], + instance_id: str, + source_path: Path, + seen_runtime_names: set[str], + emit: Callable[[str], None], +) -> dict[str, Any]: + runtime_name = _safe_name(f"asset_{instance_id}") + if runtime_name in seen_runtime_names or root.find(f"./worldbody/body[@name='{runtime_name}']") is not None: + raise ValueError(f"composition already contains runtime object: {runtime_name}") + seen_runtime_names.add(runtime_name) + transform = normalize_transform(spec.get("transform") or {}) + physics = normalize_physics(spec.get("physics") or {}) + added_dir = package_dir / "assets" / "added" / instance_id + added_dir.mkdir(parents=True, exist_ok=True) + copied_asset = added_dir / source_path.name + shutil.copy2(source_path, copied_asset) + runtime_mesh = added_dir / f"{runtime_name}.obj" + emit(f"Converting {source_path.name} into a MotrixSim composition mesh") + convert_asset_to_obj(copied_asset, runtime_mesh) + viewer_asset = copied_asset if copied_asset.suffix.lower() in {".glb", ".gltf", ".obj"} else runtime_mesh + mesh_dir = (runtime_scene.parent / str(compiler.attrib.get("meshdir") or ".")).resolve() + relative_mesh = os.path.relpath(runtime_mesh, mesh_dir).replace(os.sep, "/") + asset_node = root.find("asset") + if asset_node is None: + asset_node = ET.Element("asset") + _insert_before(root, asset_node, {"worldbody"}) + mesh_name = f"{runtime_name}_mesh" + ET.SubElement(asset_node, "mesh", {"name": mesh_name, "file": relative_mesh, "scale": _numbers(transform["scale"])}) + worldbody = root.find("worldbody") + if worldbody is None: + worldbody = ET.SubElement(root, "worldbody") + body = ET.SubElement( + worldbody, + "body", + {"name": runtime_name, "pos": _numbers(transform["translation"]), "euler": _numbers([math.radians(value) for value in transform["rotation_euler_degrees"]])}, + ) + if physics["dynamic"]: + ET.SubElement(body, "freejoint", {"name": f"{runtime_name}_free"}) + attributes = { + "name": f"{runtime_name}_geom", + "type": "mesh", + "mesh": mesh_name, + "friction": _numbers(physics["friction"]), + "contype": "0" if physics["collision_mode"] == "visual_only" else "1", + "conaffinity": "0" if physics["collision_mode"] == "visual_only" else "1", + "group": "2" if physics["collision_mode"] == "visual_only" else "3", + "rgba": _numbers(physics["rgba"]), + } + if physics["dynamic"]: + attributes["mass"] = f"{physics['mass']:.9g}" + ET.SubElement(body, "geom", attributes) + return { + "name": str(spec.get("name") or source_path.stem), + "path": str(viewer_asset), + "kind": "mesh", + "role": "added_asset", + "format": viewer_asset.suffix.lower().lstrip("."), + "size_bytes": viewer_asset.stat().st_size, + "autoload": True, + "viewer_hint": "overlay", + "catalog_id": str(spec.get("catalog_id") or ""), + "instance_id": instance_id, + "transform": transform, + "physics": physics, + "runtime": {"body": runtime_name, "validated": False}, + "added_at": _utc_now(), + } + + +def _ensure_overview_camera(root: ET.Element, *, target: str) -> None: + worldbody = root.find("worldbody") + if worldbody is None: + worldbody = ET.SubElement(root, "worldbody") + existing = worldbody.find("./camera[@name='overview']") + if existing is None: + ET.SubElement(worldbody, "camera", {"name": "overview", "mode": "targetbody", "target": target, "pos": "2 -2 1.8", "fovy": "55"}) + + +def _write_home_keyframe(runtime_scene: Path) -> None: + try: + import mujoco + import numpy as np + except ImportError as exc: + raise RuntimeError("composition keyframe generation requires mujoco and numpy") from exc + tree = ET.parse(runtime_scene) + root = tree.getroot() + keyframe = root.find("keyframe") + if keyframe is None: + keyframe = ET.SubElement(root, "keyframe") + for child in list(keyframe): + if child.tag == "key" and child.get("name") == "home": + keyframe.remove(child) + home = ET.Element("key", {"name": "home"}) + keyframe.insert(0, home) + _indent_xml(root) + tree.write(runtime_scene, encoding="utf-8", xml_declaration=True) + model = mujoco.MjModel.from_xml_path(str(runtime_scene)) + qpos = np.asarray(model.qpos0, dtype=float).copy() + arm_home = [0.0, 0.0, 0.0, -1.57079, 0.0, 1.57079] + for index, value in enumerate(arm_home, start=1): + joint_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}") + if joint_id >= 0: + qpos[int(model.jnt_qposadr[joint_id])] = value + gripper_joint = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint") + if gripper_joint >= 0: + qpos[int(model.jnt_qposadr[gripper_joint])] = -0.7853 + controls = np.zeros(model.nu, dtype=float) + controls[: min(6, model.nu)] = arm_home[: min(6, model.nu)] + if model.nu >= 7: + controls[6] = -0.7853 + home.set("qpos", _numbers(qpos.tolist())) + if model.nu: + home.set("ctrl", _numbers(controls.tolist())) + _indent_xml(root) + tree.write(runtime_scene, encoding="utf-8", xml_declaration=True) + mujoco.MjModel.from_xml_path(str(runtime_scene)) + + +def _insert_before(root: ET.Element, child: ET.Element, before_tags: set[str]) -> None: + index = next((position for position, existing in enumerate(list(root)) if existing.tag in before_tags), len(root)) + root.insert(index, child) + + def load_usd_meshes(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: try: import numpy as np diff --git a/src/real2sim_platform/rm65_controller.py b/src/real2sim_platform/rm65_controller.py index 0b5f4ea..3b63d76 100644 --- a/src/real2sim_platform/rm65_controller.py +++ b/src/real2sim_platform/rm65_controller.py @@ -51,6 +51,7 @@ class RM65PickPlaceRuntime: self._id(mujoco.mjtObj.mjOBJ_GEOM, "right_finger_pad_z_axis"), } self.target_body = self._id(mujoco.mjtObj.mjOBJ_BODY, "place_target") + self.target_home = self.model.body_pos[self.target_body].astype(float).copy() self.gripper_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint") self.gripper_qadr = int(self.model.jnt_qposadr[self.gripper_joint]) self.joint_ids = [self._id(mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}") for index in range(1, 7)] @@ -70,12 +71,20 @@ class RM65PickPlaceRuntime: def run_episode(self, *, max_steps: int) -> EpisodeResult: mujoco = self.mujoco mujoco.mj_resetDataKeyframe(self.model, self.data, 0) - cube = np.array([self.rng.uniform(0.645, 0.655), self.rng.uniform(-0.010, 0.010), 0.33], dtype=float) - target = np.array([self.rng.uniform(0.42, 0.51), self.rng.uniform(0.13, 0.22), 0.31], dtype=float) + cube_home = self.data.qpos[self.cube_qadr : self.cube_qadr + 3].astype(float).copy() + cube = cube_home + np.array([self.rng.uniform(-0.010, 0.010), self.rng.uniform(-0.010, 0.010), 0.0], dtype=float) + target = np.array( + [ + self.target_home[0] + self.rng.uniform(-0.025, 0.025), + self.target_home[1] + self.rng.uniform(-0.025, 0.025), + self.target_home[2] - 0.025, + ], + dtype=float, + ) self.data.qpos[self.cube_qadr : self.cube_qadr + 3] = cube self.data.qpos[self.cube_qadr + 3 : self.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0]) self.data.qvel[self.model.jnt_dofadr[self.cube_joint] : self.model.jnt_dofadr[self.cube_joint] + 6] = 0 - self.model.body_pos[self.target_body] = np.array([target[0], target[1], 0.335]) + self.model.body_pos[self.target_body] = np.array([target[0], target[1], self.target_home[2]]) mujoco.mj_forward(self.model, self.data) for _ in range(120): mujoco.mj_step(self.model, self.data) diff --git a/src/real2sim_platform/server.py b/src/real2sim_platform/server.py index 21edba6..73be769 100644 --- a/src/real2sim_platform/server.py +++ b/src/real2sim_platform/server.py @@ -27,7 +27,7 @@ from . import __version__ from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config -from .motrix_scene import SUPPORTED_MESH_SUFFIXES, SUPPORTED_PRIMITIVES +from .motrix_scene import COMPOSITION_ASSET_RECIPES, SUPPORTED_MESH_SUFFIXES, SUPPORTED_PRIMITIVES, composition_asset_catalog from .source_revision import read_source_revision from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file from .store import PlatformStore @@ -58,6 +58,28 @@ class SceneAssetRequest(BaseModel): revision_name: str | None = None +class SceneCompositionRobotRequest(BaseModel): + robot_id: str + transform: dict[str, Any] = Field(default_factory=dict) + + +class SceneCompositionAssetRequest(BaseModel): + catalog_id: str + instance_id: str + name: str | None = None + source: dict[str, Any] = Field(default_factory=dict) + path: str = "" + transform: dict[str, Any] = Field(default_factory=dict) + physics: dict[str, Any] = Field(default_factory=dict) + + +class SceneCompositionRequest(BaseModel): + name: str | None = None + robot: SceneCompositionRobotRequest + assets: list[SceneCompositionAssetRequest] = Field(default_factory=list) + validation_steps: int = 120 + + class CreateTaskRequest(BaseModel): type: str params: dict[str, Any] = {} @@ -307,6 +329,7 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: "motrixsim": True, "motrixsim_scene_import": True, "scene_asset_compose": True, + "scene_composition": True, }, } @@ -594,6 +617,10 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: def list_digital_assets() -> list[dict[str, Any]]: return _digital_asset_catalog(store) + @app.get("/api/robots") + def list_robots() -> list[dict[str, Any]]: + return _robot_catalog(cfg) + @app.post("/api/scenes/import") def import_scene(payload: ImportSceneRequest) -> dict[str, Any]: path = Path(payload.path) @@ -650,6 +677,27 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: "message": "数字资产合成任务已创建;通过 MotrixSim 与 MuJoCo 校验后会生成新场景版本", } + @app.post("/api/scenes/{scene_id}/compose") + def compose_scene(scene_id: str, payload: SceneCompositionRequest) -> dict[str, Any]: + try: + store.get_scene(scene_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="scene not found") from exc + composition = payload.model_dump(exclude_none=True) + params = { + "scene_id": scene_id, + "name": composition.pop("name", None), + "validation_steps": composition.pop("validation_steps", 120), + "composition": composition, + } + task = _queue_task("scene_composition", params) + return { + "ok": True, + "scene_id": scene_id, + "task": _with_task_view_model(task), + "message": "场景组合任务已创建;RM65、数字资产与 MotrixSim 运行时校验完成后会生成独立场景版本", + } + @app.patch("/api/scenes/{scene_id}") def update_scene(scene_id: str, payload: UpdateEntityRequest) -> dict[str, Any]: try: @@ -696,7 +744,7 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: payload = CreateTaskRequest(type=task_type, params=params) _validate_task_payload(payload, cfg) _validate_task_resources(store, payload) - if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation"}: + if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "scene_composition", "simulator_validation"}: raise ValueError(f"unknown task type: {payload.type}") run_dir = cfg.runs_dir / payload.type / _next_run_name(cfg.runs_dir / payload.type, payload.type) return store.create_task(task_type=payload.type, params=payload.params, run_dir=run_dir) @@ -1169,13 +1217,13 @@ def _metrics_summary(kind: str, metrics: dict[str, Any], scan: dict[str, Any], m def _resource_subtitle(kind: str, entity: dict[str, Any], manifest: dict[str, Any]) -> str: source = str(entity.get("source_type") or manifest.get("source_type") or "") version = str(entity.get("version") or "") - label = {"prepared_media": "已准备帧数据", "expert_collection": "专家采集数据", "scene_reconstruction": "场景重建", "motrixsim_spatial_bundle": "MotrixSim 空间包", "scene_revision": "场景修订", "huggingface": "HuggingFace", "upload": "上传文件", "browser_upload": "浏览器上传", "local_path": "本地导入"}.get(source, source or kind) + label = {"prepared_media": "已准备帧数据", "expert_collection": "专家采集数据", "scene_reconstruction": "场景重建", "motrixsim_spatial_bundle": "MotrixSim 空间包", "scene_revision": "场景修订", "scene_composition": "场景组合", "huggingface": "HuggingFace", "upload": "上传文件", "browser_upload": "浏览器上传", "local_path": "本地导入"}.get(source, source or kind) return f"{label}{' · ' + version if version else ''}" def _task_display_name(task: dict[str, Any]) -> str: params = task.get("params") or {} - return str(params.get("scene_name") or params.get("output_name") or _task_type_label(str(task.get("type") or ""))) + return str(params.get("scene_name") or params.get("output_name") or params.get("name") or _task_type_label(str(task.get("type") or ""))) def _task_type_label(task_type: str) -> str: @@ -1187,6 +1235,7 @@ def _task_type_label(task_type: str) -> str: "motrixsim_asset_import": "MotrixSim 资产导入", "motrixsim_scene_import": "MotrixSim 场景包导入", "scene_asset_compose": "数字资产合成", + "scene_composition": "场景组合", "simulator_validation": "仿真校验", "expert_collection": "专家采集", "training": "模型训练", @@ -1234,6 +1283,30 @@ def _validate_task_payload(payload: CreateTaskRequest, cfg: PlatformConfig) -> N raise HTTPException(status_code=400, detail="asset file not found") if not is_path_allowed(path, cfg): raise HTTPException(status_code=403, detail="asset path is outside configured import roots") + if payload.type == "scene_composition": + if not params.get("scene_id"): + raise HTTPException(status_code=400, detail="scene_composition requires scene_id") + composition = params.get("composition") or {} + robot = composition.get("robot") or {} + if robot.get("robot_id") != "rm65": + raise HTTPException(status_code=400, detail="scene_composition currently requires the registered RM65 robot") + assets = composition.get("assets") or [] + if not isinstance(assets, list) or len(assets) > 50: + raise HTTPException(status_code=400, detail="scene_composition supports at most 50 asset instances") + instance_ids = [str(item.get("instance_id") or "") for item in assets if isinstance(item, dict)] + if any(not value for value in instance_ids) or len(instance_ids) != len(set(instance_ids)): + raise HTTPException(status_code=400, detail="composition asset instance_id values must be present and unique") + for item in assets: + catalog_id = str(item.get("catalog_id") or "") + source = item.get("source") or {} + if catalog_id in COMPOSITION_ASSET_RECIPES: + continue + if catalog_id.startswith("primitive:") and catalog_id.split(":", 1)[1] in SUPPORTED_PRIMITIVES: + continue + if source.get("type") != "dataset" or not (source.get("dataset_id") and source.get("relative_path")): + raise HTTPException(status_code=400, detail=f"unregistered composition asset requires a dataset source: {catalog_id}") + if not 1 <= int(params.get("validation_steps", 120)) <= 5000: + raise HTTPException(status_code=400, detail="validation_steps must be between 1 and 5000") if payload.type == "hf_import": repo_type = params.get("repo_type", "dataset") if repo_type != "dataset": @@ -1321,6 +1394,20 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) - raise HTTPException(status_code=400, detail="资产路径越出所选数据集") if not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_MESH_SUFFIXES: raise HTTPException(status_code=400, detail="所选数据集文件不是可导入 MotrixSim 的网格资产") + if payload.type == "scene_composition": + scene = store.get_scene(str(params["scene_id"])) + package = Path(str((scene.get("metadata") or {}).get("package") or "")) + if not package.is_dir() or not (package / "scene_manifest.json").is_file(): + raise HTTPException(status_code=400, detail="所选场景不是可编辑的 MotrixSim 场景包") + for item in (params.get("composition") or {}).get("assets") or []: + source = item.get("source") or {} + if source.get("type") != "dataset": + continue + dataset = store.get_dataset(str(source.get("dataset_id") or "")) + root = Path(dataset["path"]).resolve() + candidate = (root / str(source.get("relative_path") or "")).resolve() + if candidate == root or root not in candidate.parents or not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_MESH_SUFFIXES: + raise HTTPException(status_code=400, detail="组合资产不是数据集内可导入 MotrixSim 的网格文件") if payload.type == "training" and params.get("dataset_id"): dataset = store.get_dataset(str(params["dataset_id"])) source = str(dataset.get("source_type") or "") @@ -1342,6 +1429,15 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) - def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]: primitive_names = {"box": "立方体", "sphere": "球体", "cylinder": "圆柱体", "capsule": "胶囊体"} items: list[dict[str, Any]] = [ + { + **recipe, + "source_type": "recipe", + "format": "compound" if len(recipe.get("components") or []) > 1 else str((recipe.get("components") or [{}])[0].get("primitive") or "primitive"), + "source": {"type": "recipe", "recipe_id": recipe["id"]}, + "capabilities": {"motrixsim_physics": True, "browser_preview": True, "dynamic": bool((recipe.get("physics") or {}).get("dynamic")), "collision": str((recipe.get("physics") or {}).get("collision_mode")) != "visual_only"}, + } + for recipe in composition_asset_catalog() + ] + [ { "id": f"primitive:{primitive}", "name": primitive_names.get(primitive, primitive), @@ -1379,6 +1475,7 @@ def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]: "relative_path": relative.as_posix(), "format": suffix.lstrip("."), "size_bytes": candidate.stat().st_size, + "preview_url": _workspace_public_url(candidate) if suffix in {".obj", ".glb", ".gltf", ".ply"} else "", "source": {"type": "dataset", "dataset_id": dataset["id"], "relative_path": relative.as_posix()}, "capabilities": { "motrixsim_physics": True, @@ -1391,6 +1488,30 @@ def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]: return items +def _robot_catalog(cfg: PlatformConfig) -> list[dict[str, Any]]: + return [ + { + "id": "rm65", + "name": "RM65 双指夹爪", + "manufacturer": "睿尔曼", + "simulator": "motrixsim", + "description": "6 轴机械臂、双指夹爪、7 个位置执行器和真实碰撞体", + "preview_url": "/static/assets/robots/rm65.glb", + "controller": "rm65_pick_place_6d_ik", + "joint_count": 6, + "actuator_count": 7, + "payload_kg": 5, + "default_transform": {"translation": [1.03, 0.0, 0.011], "rotation_euler_degrees": [0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}, + "source": { + "repository": "Motphys/motrixsim-docs", + "commit": os.environ.get("REAL2SIM_MOTRIXSIM_DOCS_COMMIT", "2beecea67c81d25e29cefa30a80c0ddad7b44dde"), + "mjcf": "examples/assets/rm65_open_cabinet/mjx_rm65.xml", + }, + "capabilities": {"motrixsim": True, "mujoco": True, "browser_preview": True, "pick_place": True}, + } + ] + + def _active_admin_count(store: PlatformStore) -> int: return sum(1 for user in store.list_users() if user["role"] == "admin" and not user["disabled"]) @@ -1512,6 +1633,7 @@ def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]: "gsplat_python": str(cfg.gsplat_python), "cuda_home": str(cfg.cuda_home), "runtime_env_script": str(cfg.runtime_env_script), + "motrixsim_docs_root": str(cfg.motrixsim_docs_root), "discoverse_commit": cfg.discoverse_commit, "act_commit": cfg.act_commit, "anysplat_commit": cfg.anysplat_commit, diff --git a/src/real2sim_platform/static/app.js b/src/real2sim_platform/static/app.js index 94372a3..0ff4466 100644 --- a/src/real2sim_platform/static/app.js +++ b/src/real2sim_platform/static/app.js @@ -1,4 +1,4 @@ -const STATIC_VERSION = "20260712-motrix-spatial-v4"; +const STATIC_VERSION = "20260712-scene-composition-v1"; const state = { datasets: [], @@ -11,6 +11,7 @@ const state = { editing: null, assetLibrary: [], digitalAssets: [], + robots: [], refreshTimer: null, taskEvents: null, selections: {}, @@ -19,6 +20,21 @@ const state = { recipes: null, initialized: false, pendingPassword: "", + composition: { + baseSceneId: "", + robotId: "", + robotTransform: null, + assets: [], + selectedId: "", + tool: "select", + history: [], + future: [], + bounds: null, + previewSceneId: "", + savedSceneId: "", + collectionReady: false, + pendingTaskId: "", + }, }; const $ = (id) => document.getElementById(id); @@ -26,6 +42,7 @@ const $ = (id) => document.getElementById(id); const taskScopes = { dataTaskList: ["hf_import", "media_prepare"], sceneTaskList: ["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"], + compositionTaskList: ["scene_composition", "simulator_validation"], collectTaskList: ["expert_collection"], trainTaskList: ["training"], inferTaskList: ["inference"], @@ -90,6 +107,7 @@ function startTaskEvents() { state.tasks = JSON.parse(event.data); renderScopedTaskLists(); renderOverview(); + renderComposition(); if (state.selectedTaskId && $("taskDrawer")?.classList.contains("open")) showTask(state.selectedTaskId, { preservePreview: true }).catch(console.warn); if (state.tasks.some((task) => terminalStatus(task.status) && !previousTerminal.has(task.id))) refresh({ quiet: true }).catch(console.warn); }); @@ -125,18 +143,20 @@ function fillRecipeSelect(id, items, emptyLabel) { async function refresh({ quiet = false } = {}) { try { - const [datasets, scenes, models, tasks, digitalAssets] = await Promise.all([ + const [datasets, scenes, models, tasks, digitalAssets, robots] = await Promise.all([ api("/api/datasets"), api("/api/scenes"), api("/api/models"), api("/api/tasks"), api("/api/assets"), + api("/api/robots"), ]); state.datasets = Array.isArray(datasets) ? datasets : []; state.scenes = Array.isArray(scenes) ? scenes : []; state.models = Array.isArray(models) ? models : []; state.tasks = Array.isArray(tasks) ? tasks : []; state.digitalAssets = Array.isArray(digitalAssets) ? digitalAssets : []; + state.robots = Array.isArray(robots) ? robots : []; if (state.activeSceneId && !state.scenes.some((scene) => scene.id === state.activeSceneId)) { state.activeSceneId = ""; } @@ -159,6 +179,7 @@ function renderAll() { renderDatasetList(); renderReconstructDatasetCards(); renderSceneList(); + renderComposition(); renderPreflightSummary(); renderCollectionDatasets(); renderModelList(); @@ -175,10 +196,11 @@ function updateActionAvailability() { setActionAvailability("createMediaPrepare", writable && state.datasets.length > 0, writable ? "请先新增数据集" : "只读用户不能创建任务"); setActionAvailability("createPreflightTask", writable && preparedDatasets().length > 0, "请先准备帧数据"); setActionAvailability("createSceneTask", writable && preparedDatasets().length > 0, "请先准备帧数据"); - setActionAvailability("createCollectTask", writable && state.scenes.length > 0 && Boolean($("policyPreset")?.value), "请先选择兼容真实控制器的场景"); + setActionAvailability("createCollectTask", writable && collectionReadyScenes().length > 0 && Boolean($("policyPreset")?.value), "请先保存通过运行时校验的 RM65 组合场景"); setActionAvailability("createTrainTask", writable && collectionDatasets().length > 0 && Boolean($("trainRecipe")?.value), "请先完成真实专家采集并确认 ACT 运行时"); setActionAvailability("createInferTask", writable && state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型"); setActionAvailability("addSceneAsset", writable && state.scenes.length > 0, "请先完成场景重建"); + setActionAvailability("saveSceneComposition", writable && Boolean(state.composition.baseSceneId && state.composition.robotId), "请先选择基础场景和机器人"); } function setActionAvailability(id, enabled, reason) { @@ -192,16 +214,19 @@ function switchTab(tabId) { document.querySelectorAll(".tab").forEach((button) => button.classList.toggle("active", button.dataset.tab === tabId)); document.querySelectorAll(".page").forEach((page) => page.classList.toggle("active", page.id === tabId)); refreshSelectionBars(); + if (tabId === "compose") renderComposition(); } function renderSelectors() { fillSelect("prepareDataset", state.datasets, "暂无数据集,请先新增"); fillSelect("reconstructInput", preparedDatasets(), "暂无已准备帧数据,请先在数据管理中准备"); - fillSelect("collectScene", state.scenes, "暂无场景,请先重建"); + fillSelect("collectScene", collectionReadyScenes(), "暂无可采集组合场景,请先完成场景组合"); fillSelect("trainDataset", collectionDatasets(), "暂无采集数据,请先采集"); fillSelect("inferScene", state.scenes, "暂无场景,请先重建"); fillSelect("inferModel", state.models, "暂无模型,请先训练"); fillSelect("assetScene", state.scenes, "暂无场景,请先重建"); + fillSelect("composeBaseScene", baseCompositionScenes(), "暂无可编辑场景,请先导入或重建"); + if ($("composeBaseScene") && state.composition.baseSceneId) $("composeBaseScene").value = state.composition.baseSceneId; const assetScene = $("assetScene"); if (assetScene && state.activeSceneId && state.scenes.some((scene) => scene.id === state.activeSceneId)) { assetScene.value = state.activeSceneId; @@ -233,6 +258,7 @@ function renderOverview() { ["数据导入", state.datasets.length > 0], ["输入准备", state.datasets.some((item) => item.source_type === "prepared_media")], ["场景重建", state.scenes.length > 0], + ["场景组合", collectionReadyScenes().length > 0], ["数据采集", collectionDatasets().length > 0], ["模型训练", state.models.length > 0], ["在线推理", state.tasks.some((task) => task.type === "inference" && task.status === "succeeded")], @@ -262,9 +288,10 @@ function nextAction() { if (!state.datasets.length) return { index: 0, label: "新增数据", text: "先导入 HuggingFace 样例、上传文件或本地目录。", action: () => switchTab("data") }; if (!state.datasets.some((item) => item.source_type === "prepared_media")) return { index: 1, label: "准备输入", text: "已有数据集,下一步是抽帧并生成 prepared frames。", action: () => switchTab("data") }; if (!state.scenes.length) return { index: 2, label: "开始重建", text: "已有准备数据,建议先预检,再启动真实重建。", action: () => switchTab("reconstruct") }; - if (!collectionDatasets().length) return { index: 3, label: "采集数据", text: "已有场景,可用专家策略采集训练数据。", action: () => switchTab("collect") }; - if (!state.models.length) return { index: 4, label: "训练模型", text: "已有采集数据,可训练策略模型。", action: () => switchTab("train") }; - return { index: 5, label: "运行推理", text: "闭环资源已就绪,可以选择模型和场景在线推理。", action: () => switchTab("infer") }; + if (!collectionReadyScenes().length) return { index: 3, label: "组合场景", text: "在重建场景中加入 RM65、任务资产和 MotrixSim 物理属性。", action: () => switchTab("compose") }; + if (!collectionDatasets().length) return { index: 4, label: "采集数据", text: "组合场景已通过运行时校验,可以启动真实专家采集。", action: () => switchTab("collect") }; + if (!state.models.length) return { index: 5, label: "训练模型", text: "已有采集数据,可训练策略模型。", action: () => switchTab("train") }; + return { index: 6, label: "运行推理", text: "闭环资源已就绪,可以选择模型和场景在线推理。", action: () => switchTab("infer") }; } function renderDatasetList() { @@ -327,6 +354,373 @@ function renderSceneList() { setHtml("sceneList", rows || emptyState("暂无场景,请先在“场景重建”中创建。")); } +function baseCompositionScenes() { + return state.scenes.filter((scene) => scene?.metadata?.editable !== false); +} + +function collectionReadyScenes() { + return state.scenes.filter((scene) => scene?.metadata?.collection_ready || (scene.tags || []).includes("collection-ready")); +} + +function ensureCompositionDraft() { + const draft = state.composition; + const scenes = baseCompositionScenes(); + if (!scenes.some((scene) => scene.id === draft.baseSceneId)) { + draft.baseSceneId = scenes[0]?.id || ""; + draft.bounds = null; + draft.previewSceneId = ""; + } + if (!state.robots.some((robot) => robot.id === draft.robotId)) { + const robot = state.robots[0]; + draft.robotId = robot?.id || ""; + draft.robotTransform = robot ? cloneValue(robot.default_transform) : null; + } + const validIds = new Set([`robot:${draft.robotId}`, ...draft.assets.map((asset) => asset.instance_id)]); + if (!validIds.has(draft.selectedId)) draft.selectedId = draft.robotId ? `robot:${draft.robotId}` : draft.assets[0]?.instance_id || ""; +} + +function renderComposition() { + if (!$("composeRobotList")) return; + ensureCompositionDraft(); + const draft = state.composition; + const baseScene = state.scenes.find((scene) => scene.id === draft.baseSceneId); + const baseSelect = $("composeBaseScene"); + if (baseSelect && baseScene) baseSelect.value = baseScene.id; + + setHtml("composeRobotList", state.robots.map((robot) => ` + `).join("") || emptyState("暂无已注册机器人。")); + + const visibleAssets = state.digitalAssets.filter((asset) => asset.source_type === "recipe" || asset.source_type === "primitive" || asset.capabilities?.browser_preview).slice(0, 30); + setHtml("composeAssetList", visibleAssets.map((asset) => { + const inScene = draft.assets.some((item) => item.catalog_id === asset.id); + return ``; + }).join("") || emptyState("暂无可用于组合的数字资产。")); + + const hierarchy = [ + baseScene ? { id: "base-scene", name: resourceDisplayName(baseScene), type: "基础场景", removable: false } : null, + draft.robotId ? { id: `robot:${draft.robotId}`, name: state.robots.find((robot) => robot.id === draft.robotId)?.name || draft.robotId, type: "机器人", removable: true } : null, + ...draft.assets.map((asset) => ({ id: asset.instance_id, name: asset.name, type: compositionAssetType(asset), removable: true })), + ].filter(Boolean); + setText("compositionObjectCount", `${hierarchy.length} 个对象`); + setHtml("compositionHierarchy", hierarchy.map((item) => ` +
+ ${escapeHtml(item.name)}${escapeHtml(item.type)}${item.removable ? `` : ""} +
`).join("") || emptyState("先选择基础场景和机器人。")); + + renderCompositionInspector(); + renderCompositionPreview(baseScene); + const requiredReady = ["recipe:coffee_table", "recipe:pick_cube", "recipe:place_target"].every((id) => draft.assets.some((asset) => asset.catalog_id === id)); + const pending = state.tasks.find((task) => task.id === draft.pendingTaskId); + if (pending?.status === "succeeded" && pending.result?.scene_id) { + draft.savedSceneId = pending.result.scene_id; + draft.collectionReady = Boolean(pending.result.collection_ready); + draft.pendingTaskId = ""; + } + const savedScene = state.scenes.find((scene) => scene.id === draft.savedSceneId); + const motrixReady = Boolean(state.capabilities?.runtimes?.motrixsim?.ready); + setHtml("composeCompatibility", ` + ${motrixReady ? "MotrixSim 运行时已就绪" : "MotrixSim 运行时不可用"} + RM65:${draft.robotId ? "已加入" : "未选择"} + 采集对象:${requiredReady ? "茶几、抓取方块、放置目标齐全" : "还需加入茶几、抓取方块和放置目标"} + 保存状态:${savedScene ? `已生成 ${escapeHtml(resourceDisplayName(savedScene))}` : pending ? escapeHtml(statusLabel(pending.status)) : "尚未保存"}`); + setText("composeStageStatus", baseScene ? `${resourceDisplayName(baseScene)} · ${draft.assets.length} 个数字资产 · ${draft.robotId ? "RM65 已加入" : "未选择机器人"}` : "等待基础场景"); + setActionAvailability("saveSceneComposition", state.user?.role !== "viewer" && Boolean(baseScene && draft.robotId && motrixReady), motrixReady ? "请先选择基础场景和机器人" : "MotrixSim worker 尚未就绪"); + setActionAvailability("testSceneComposition", Boolean(savedScene), "请先保存并等待组合场景通过运行时校验"); + setActionAvailability("collectSceneComposition", Boolean(savedScene && draft.collectionReady), "组合场景需包含茶几、抓取方块和放置目标并通过校验"); +} + +function renderCompositionPreview(baseScene) { + const iframe = $("compositionPreview"); + if (!iframe) return; + if (!baseScene) { + iframe.removeAttribute("src"); + return; + } + if (state.composition.previewSceneId !== baseScene.id || !iframe.src.includes(STATIC_VERSION)) { + state.composition.previewSceneId = baseScene.id; + iframe.src = `/static/scene-viewer.html?v=${STATIC_VERSION}&embedded=1&composer=1&scene_id=${encodeURIComponent(baseScene.id)}`; + iframe.onload = () => postCompositionDraft(); + loadCompositionBounds(baseScene.id).catch((error) => console.warn(error)); + } else { + postCompositionDraft(); + } +} + +async function loadCompositionBounds(sceneId) { + const manifest = await api(`/api/scenes/${encodeURIComponent(sceneId)}/manifest`); + if (state.composition.baseSceneId !== sceneId) return; + const min = Array.isArray(manifest.bounds_m?.min) ? manifest.bounds_m.min.map(Number) : []; + const max = Array.isArray(manifest.bounds_m?.max) ? manifest.bounds_m.max.map(Number) : []; + state.composition.bounds = min.length === 3 && max.length === 3 && [...min, ...max].every(Number.isFinite) ? { min, max } : null; +} + +function postCompositionDraft() { + const iframe = $("compositionPreview"); + if (!iframe?.contentWindow) return; + const draft = state.composition; + const robot = state.robots.find((item) => item.id === draft.robotId); + const robotPayload = robot ? { + ...robot, + instance_id: `robot:${robot.id}`, + role: "robot", + kind: "robot", + name: "rm65.glb", + url: robot.preview_url, + transform: cloneValue(draft.robotTransform), + } : null; + const assets = draft.assets.map((instance) => { + const catalog = state.digitalAssets.find((item) => item.id === instance.catalog_id) || {}; + const components = Array.isArray(catalog.components) ? catalog.components.map((component) => ({ + name: component.name, + viewer_primitive: component.primitive, + translation: component.position || [0, 0, 0], + rotation_euler_degrees: component.rotation_euler_degrees || [0, 0, 0], + dimensions: component.dimensions || catalog.physics?.dimensions || [0.1, 0.1, 0.1], + rgba: component.rgba || catalog.physics?.rgba || [0.13, 0.76, 0.67, 1], + })) : []; + return { + ...catalog, + ...cloneValue(instance), + role: "composition_draft", + url: catalog.preview_url || catalog.url || "", + viewer_primitive: components.length === 1 ? components[0].viewer_primitive : catalog.viewer_primitive, + viewer_components: components, + }; + }); + iframe.contentWindow.postMessage({ + type: "real2sim-composition-draft", + robot: robotPayload, + assets, + selected_id: draft.selectedId, + tool: draft.tool, + }, location.origin); +} + +function selectedCompositionObject() { + const draft = state.composition; + if (draft.selectedId === `robot:${draft.robotId}`) { + const robot = state.robots.find((item) => item.id === draft.robotId); + return robot ? { type: "robot", id: draft.selectedId, name: robot.name, transform: draft.robotTransform, robot } : null; + } + const asset = draft.assets.find((item) => item.instance_id === draft.selectedId); + return asset ? { type: "asset", id: asset.instance_id, name: asset.name, transform: asset.transform, physics: asset.physics, asset } : null; +} + +function renderCompositionInspector() { + const selected = selectedCompositionObject(); + setHtml("compositionSelection", selected + ? `${escapeHtml(selected.name)}${selected.type === "robot" ? "RM65 机器人 · 关节与惯性保持标定尺度" : escapeHtml(compositionAssetType(selected.asset))}` + : "选择画布或层级中的对象"); + const transform = selected?.transform || { translation: [0, 0, 0], rotation_euler_degrees: [0, 0, 0], scale: [1, 1, 1] }; + const values = { + composePosX: transform.translation?.[0] ?? 0, + composePosY: transform.translation?.[1] ?? 0, + composePosZ: transform.translation?.[2] ?? 0, + composeRotX: transform.rotation_euler_degrees?.[0] ?? 0, + composeRotY: transform.rotation_euler_degrees?.[1] ?? 0, + composeRotZ: transform.rotation_euler_degrees?.[2] ?? 0, + composeScaleX: transform.scale?.[0] ?? 1, + composeScaleY: transform.scale?.[1] ?? 1, + composeScaleZ: transform.scale?.[2] ?? 1, + composeMass: selected?.physics?.mass ?? selected?.robot?.payload_kg ?? 0, + }; + Object.entries(values).forEach(([id, value]) => { + const input = $(id); + if (input && document.activeElement !== input) input.value = Number(value).toFixed(id.startsWith("composeRot") ? 1 : 3); + }); + if ($("composeDynamic")) $("composeDynamic").checked = Boolean(selected?.physics?.dynamic); + if ($("composeCollision")) $("composeCollision").value = selected?.physics?.collision_mode || "visual_and_collision"; + const disabled = !selected; + ["composePosX", "composePosY", "composePosZ", "composeRotX", "composeRotY", "composeRotZ"].forEach((id) => { if ($(id)) $(id).disabled = disabled; }); + ["composeScaleX", "composeScaleY", "composeScaleZ", "composeDynamic", "composeMass", "composeCollision"].forEach((id) => { if ($(id)) $(id).disabled = disabled || selected?.type === "robot"; }); + if ($("removeCompositionObject")) $("removeCompositionObject").disabled = !selected; +} + +function compositionAssetDescription(asset) { + if (asset.source_type === "primitive") return `${sourceTypeLabel(asset.format)} · 可设置碰撞和动态属性`; + if (asset.source_type === "dataset") return `${asset.dataset_name || "数据集"} · ${String(asset.format || "mesh").toUpperCase()}`; + return "MotrixSim 数字资产"; +} + +function compositionAssetType(asset) { + const catalog = state.digitalAssets.find((item) => item.id === asset?.catalog_id); + if (catalog?.source_type === "recipe") return catalog.category === "task_object" ? "动态任务物体" : catalog.category === "task_marker" ? "任务目标" : "组合资产"; + return catalog?.source_type === "dataset" ? "已导入网格" : "基础几何"; +} + +function cloneValue(value) { + return value == null ? value : JSON.parse(JSON.stringify(value)); +} + +function compositionSnapshot() { + const draft = state.composition; + return cloneValue({ robotId: draft.robotId, robotTransform: draft.robotTransform, assets: draft.assets, selectedId: draft.selectedId }); +} + +function pushCompositionHistory() { + const draft = state.composition; + draft.history.push(compositionSnapshot()); + if (draft.history.length > 40) draft.history.shift(); + draft.future = []; +} + +function restoreCompositionSnapshot(snapshot) { + if (!snapshot) return; + Object.assign(state.composition, cloneValue(snapshot), { savedSceneId: "", collectionReady: false, pendingTaskId: "" }); + renderComposition(); +} + +function markCompositionDirty() { + Object.assign(state.composition, { savedSceneId: "", collectionReady: false, pendingTaskId: "" }); +} + +function addCompositionAsset(catalogId, position = null, { record = true } = {}) { + const catalog = state.digitalAssets.find((asset) => asset.id === catalogId); + if (!catalog) return toast("所选数字资产已不在资源目录中"); + if (catalog.source_type === "recipe") { + const existing = state.composition.assets.find((asset) => asset.catalog_id === catalogId); + if (existing) { + state.composition.selectedId = existing.instance_id; + renderComposition(); + return toast("该任务资产已在场景中,可直接调整位置"); + } + } + if (record) pushCompositionHistory(); + const transform = cloneValue(catalog.default_transform || { translation: [0, 0, 0.1], rotation_euler_degrees: [0, 0, 0], scale: [1, 1, 1] }); + if (position) { + transform.translation[0] = position[0]; + transform.translation[1] = position[1]; + } + const instance = { + catalog_id: catalog.id, + instance_id: `asset_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 7)}`, + name: catalog.name, + source: cloneValue(catalog.source || {}), + transform, + physics: cloneValue(catalog.physics || { dynamic: false, mass: 0.25, collision_mode: "visual_and_collision", dimensions: [0.1, 0.1, 0.1] }), + }; + state.composition.assets.push(instance); + state.composition.selectedId = instance.instance_id; + markCompositionDirty(); + renderComposition(); +} + +function compositionDropPosition(event) { + const bounds = state.composition.bounds; + const catalog = state.digitalAssets.find((asset) => asset.id === document.body.dataset.compositionCatalog); + const fallback = cloneValue(catalog?.default_transform?.translation || [0, 0, 0.1]); + if (!bounds) return fallback; + const rect = $("compositionDropOverlay").getBoundingClientRect(); + const nx = Math.max(0, Math.min(1, (event.clientX - rect.left) / Math.max(1, rect.width))); + const ny = Math.max(0, Math.min(1, (event.clientY - rect.top) / Math.max(1, rect.height))); + const width = Math.min(6, Math.max(0.4, bounds.max[0] - bounds.min[0])); + const depth = Math.min(6, Math.max(0.4, bounds.max[1] - bounds.min[1])); + const centerX = (bounds.min[0] + bounds.max[0]) / 2; + const centerY = (bounds.min[1] + bounds.max[1]) / 2; + return [centerX + (0.5 - ny) * width * 0.7, centerY + (nx - 0.5) * depth * 0.7, fallback[2]]; +} + +function selectCompositionObject(id) { + state.composition.selectedId = id; + renderComposition(); +} + +function removeCompositionObject(id = state.composition.selectedId) { + if (!id) return; + pushCompositionHistory(); + if (id.startsWith("robot:")) { + state.composition.robotId = ""; + state.composition.robotTransform = null; + } else { + state.composition.assets = state.composition.assets.filter((asset) => asset.instance_id !== id); + } + state.composition.selectedId = state.composition.robotId ? `robot:${state.composition.robotId}` : state.composition.assets[0]?.instance_id || ""; + markCompositionDirty(); + renderComposition(); +} + +function updateSelectedCompositionFromForm() { + const selected = selectedCompositionObject(); + if (!selected) return; + pushCompositionHistory(); + selected.transform.translation = [Number($("composePosX").value), Number($("composePosY").value), Number($("composePosZ").value)]; + selected.transform.rotation_euler_degrees = [Number($("composeRotX").value), Number($("composeRotY").value), Number($("composeRotZ").value)]; + if (selected.type !== "robot") { + selected.transform.scale = [Number($("composeScaleX").value), Number($("composeScaleY").value), Number($("composeScaleZ").value)]; + selected.physics.dynamic = Boolean($("composeDynamic").checked); + selected.physics.mass = Number($("composeMass").value); + selected.physics.collision_mode = $("composeCollision").value; + } + markCompositionDirty(); + renderComposition(); +} + +function loadPickPlacePreset() { + pushCompositionHistory(); + for (const catalog of state.digitalAssets.filter((asset) => asset.source_type === "recipe" && asset.required_for_collection)) { + if (!state.composition.assets.some((asset) => asset.catalog_id === catalog.id)) addCompositionAsset(catalog.id, null, { record: false }); + } + markCompositionDirty(); + renderComposition(); +} + +async function saveSceneComposition() { + const draft = state.composition; + if (!draft.baseSceneId || !draft.robotId) return toast("请先选择基础场景和机器人"); + const name = $("compositionName")?.value.trim() || "RM65 组合场景"; + setBusy(true, "正在创建 MotrixSim 场景组合任务"); + try { + const result = await api(`/api/scenes/${encodeURIComponent(draft.baseSceneId)}/compose`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name, + robot: { robot_id: draft.robotId, transform: cloneValue(draft.robotTransform) }, + assets: draft.assets.map((asset) => ({ + catalog_id: asset.catalog_id, + instance_id: asset.instance_id, + name: asset.name, + source: cloneValue(asset.source || {}), + transform: cloneValue(asset.transform), + physics: cloneValue(asset.physics || {}), + })), + validation_steps: 120, + }), + }); + draft.pendingTaskId = result.task.id; + draft.savedSceneId = ""; + draft.collectionReady = false; + state.selectedTaskId = result.task.id; + toast("组合任务已创建,正在执行真实 MotrixSim 校验"); + await refresh({ quiet: true }); + showTask(result.task.id).catch((error) => toast(error.message)); + } finally { + setBusy(false); + } +} + +async function testSceneComposition() { + const sceneId = state.composition.savedSceneId; + if (!sceneId) return toast("请先保存组合场景"); + await createTask("simulator_validation", { scene_id: sceneId, simulators: ["mujoco", "motrixsim"], level: "runtime_required" }); + switchTab("compose"); +} + +function collectSceneComposition() { + const sceneId = state.composition.savedSceneId; + if (!sceneId || !state.composition.collectionReady) return toast("该组合场景尚未满足采集对象约定"); + switchTab("collect"); + renderSelectors(); + if ($("collectScene")) $("collectScene").value = sceneId; + toast("已选择组合场景,可配置 Episode 数后开始采集"); +} + function renderPreflightSummary() { const node = $("preflightSummary"); if (!node) return; @@ -1091,7 +1485,7 @@ function collectionDatasets() { } function sceneIdFromTask(task) { - return ["scene_reconstruction", "motrixsim_scene_import", "scene_asset_compose"].includes(task?.type) ? task.result?.scene_id || "" : ""; + return ["scene_reconstruction", "motrixsim_scene_import", "scene_asset_compose", "scene_composition"].includes(task?.type) ? task.result?.scene_id || "" : ""; } function resourceView(kind, item) { @@ -1187,7 +1581,7 @@ function taskSubtitle(task) { function taskProgress(task) { if (task.status === "succeeded") return 100; if (Number.isFinite(Number(task.progress))) return Math.max(0, Math.min(100, Number(task.progress))); - return { succeeded: 100, running: 50, queued: 8, canceling: 45 }[task.status] || 0; + return 0; } function taskStage(task) { @@ -1205,6 +1599,7 @@ function taskTypeLabel(type) { motrixsim_asset_import: "MotrixSim 资产导入", motrixsim_scene_import: "MotrixSim 场景包导入", scene_asset_compose: "数字资产合成", + scene_composition: "场景组合", simulator_validation: "仿真校验", renderer_benchmark: "渲染器对比", scene_cover_render: "场景封面", @@ -1253,6 +1648,9 @@ function sourceTypeLabel(type) { scene_reconstruction: "场景重建", motrixsim_spatial_bundle: "MotrixSim 空间包", scene_revision: "场景修订", + scene_composition: "场景组合", + recipe: "任务资产配方", + primitive: "基础几何", training: "训练模型", inference: "推理结果", splat_or_pointcloud: "点云 / Splat", @@ -1377,7 +1775,7 @@ function assetPriority(asset) { } function assetRoleLabel(role) { - return ({ added_asset: "新增资产", collision: "碰撞体", visual: "可视化", texture: "纹理", simulator_import: "仿真导入", package: "场景包", source: "来源资产" })[role] || role; + return ({ robot: "机器人", added_asset: "新增资产", collision: "碰撞体", visual: "可视化", texture: "纹理", simulator_import: "仿真导入", package: "场景包", source: "来源资产" })[role] || role; } function bulkMessage(result, label) { @@ -1396,6 +1794,7 @@ function exportSummary(payload = {}) { function switchTaskPage(type) { if (["hf_import", "media_prepare"].includes(type)) switchTab("data"); + else if (type === "scene_composition") switchTab("compose"); else if (["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"].includes(type)) switchTab("reconstruct"); else if (type === "expert_collection") switchTab("collect"); else if (type === "training") switchTab("train"); @@ -1599,6 +1998,43 @@ function bindEvents() { $("reconstructInput")?.addEventListener("change", () => { renderReconstructDatasetCards(); }); + $("composeBaseScene")?.addEventListener("change", () => { + pushCompositionHistory(); + state.composition.baseSceneId = $("composeBaseScene").value; + state.composition.bounds = null; + state.composition.previewSceneId = ""; + markCompositionDirty(); + renderComposition(); + }); + $("loadPickPlacePreset")?.addEventListener("click", loadPickPlacePreset); + $("saveSceneComposition")?.addEventListener("click", saveSceneComposition); + $("testSceneComposition")?.addEventListener("click", testSceneComposition); + $("collectSceneComposition")?.addEventListener("click", collectSceneComposition); + $("removeCompositionObject")?.addEventListener("click", () => removeCompositionObject()); + ["composePosX", "composePosY", "composePosZ", "composeRotX", "composeRotY", "composeRotZ", "composeScaleX", "composeScaleY", "composeScaleZ", "composeDynamic", "composeMass", "composeCollision"].forEach((id) => { + $(id)?.addEventListener("change", updateSelectedCompositionFromForm); + }); + $("composeUndo")?.addEventListener("click", () => { + const snapshot = state.composition.history.pop(); + if (!snapshot) return toast("没有可撤销的操作"); + state.composition.future.push(compositionSnapshot()); + restoreCompositionSnapshot(snapshot); + }); + $("composeRedo")?.addEventListener("click", () => { + const snapshot = state.composition.future.pop(); + if (!snapshot) return toast("没有可重做的操作"); + state.composition.history.push(compositionSnapshot()); + restoreCompositionSnapshot(snapshot); + }); + $("compositionDropOverlay")?.addEventListener("dragover", (event) => event.preventDefault()); + $("compositionDropOverlay")?.addEventListener("drop", (event) => { + event.preventDefault(); + const catalogId = document.body.dataset.compositionCatalog; + const robotId = document.body.dataset.compositionRobot; + if (catalogId) addCompositionAsset(catalogId, compositionDropPosition(event)); + if (robotId) selectCompositionRobot(robotId, compositionDropPosition(event)); + clearCompositionDrag(); + }); $("createMediaPrepare")?.addEventListener("click", async () => { const datasetId = $("prepareDataset")?.value || state.datasets[0]?.id; @@ -1649,8 +2085,8 @@ function bindEvents() { }); $("createCollectTask")?.addEventListener("click", async () => { - const sceneId = $("collectScene")?.value || state.scenes[0]?.id; - if (!sceneId) return toast("请先完成场景重建。"); + const sceneId = $("collectScene")?.value || collectionReadyScenes()[0]?.id; + if (!sceneId) return toast("请先保存满足 RM65 控制器约定的组合场景。"); await createTask("expert_collection", { scene_id: sceneId, policy_preset: $("policyPreset")?.value, @@ -1692,10 +2128,93 @@ function bindEvents() { } }); + document.body.addEventListener("dragstart", (event) => { + const asset = event.target.closest("[data-compose-catalog]"); + const robot = event.target.closest("[data-compose-robot]"); + if (!asset && !robot) return; + document.body.dataset.compositionCatalog = asset?.dataset.composeCatalog || ""; + document.body.dataset.compositionRobot = robot?.dataset.composeRobot || ""; + document.body.classList.add("composition-dragging"); + event.dataTransfer.effectAllowed = "copy"; + event.dataTransfer.setData("text/plain", asset?.dataset.composeCatalog || robot?.dataset.composeRobot || "real2sim-resource"); + }); + document.body.addEventListener("dragend", clearCompositionDrag); + document.body.addEventListener("click", handleBodyClick); + window.addEventListener("message", handleCompositionViewerMessage); +} + +function clearCompositionDrag() { + document.body.classList.remove("composition-dragging"); + delete document.body.dataset.compositionCatalog; + delete document.body.dataset.compositionRobot; +} + +function selectCompositionRobot(robotId, position = null) { + const robot = state.robots.find((item) => item.id === robotId); + if (!robot) return toast("机器人已不在资源目录中"); + pushCompositionHistory(); + state.composition.robotId = robot.id; + state.composition.robotTransform = cloneValue(robot.default_transform); + if (position) { + state.composition.robotTransform.translation[0] = position[0]; + state.composition.robotTransform.translation[1] = position[1]; + } + state.composition.selectedId = `robot:${robot.id}`; + markCompositionDirty(); + renderComposition(); +} + +function handleCompositionViewerMessage(event) { + const iframe = $("compositionPreview"); + if (event.origin !== location.origin || event.source !== iframe?.contentWindow || !event.data) return; + const payload = event.data; + if (payload.type === "real2sim-composition-selected") { + state.composition.selectedId = String(payload.instance_id || ""); + renderCompositionInspector(); + return; + } + if (payload.type === "real2sim-composition-transform-start") { + pushCompositionHistory(); + return; + } + if (payload.type === "real2sim-composition-transform") { + const id = String(payload.instance_id || ""); + const transform = cloneValue(payload.transform || {}); + if (id === `robot:${state.composition.robotId}`) state.composition.robotTransform = transform; + else { + const asset = state.composition.assets.find((item) => item.instance_id === id); + if (asset) asset.transform = transform; + } + state.composition.selectedId = id; + markCompositionDirty(); + renderCompositionInspector(); + setText("composeStageStatus", "对象位置已更新,保存后将写入 MotrixSim MJCF"); + return; + } + if (payload.type === "real2sim-composition-transform-end") renderComposition(); } async function handleBodyClick(event) { + const composeRobot = event.target.closest("[data-compose-robot]"); + if (composeRobot) selectCompositionRobot(composeRobot.dataset.composeRobot); + + const composeCatalog = event.target.closest("[data-compose-catalog]"); + if (composeCatalog) addCompositionAsset(composeCatalog.dataset.composeCatalog); + + const composeSelect = event.target.closest("[data-compose-select]"); + if (composeSelect && !event.target.closest("button")) selectCompositionObject(composeSelect.dataset.composeSelect); + + const composeRemove = event.target.closest("[data-compose-remove]"); + if (composeRemove) removeCompositionObject(composeRemove.dataset.composeRemove); + + const composeTool = event.target.closest("[data-compose-tool]"); + if (composeTool) { + state.composition.tool = composeTool.dataset.composeTool; + document.querySelectorAll("[data-compose-tool]").forEach((button) => button.classList.toggle("active", button === composeTool)); + postCompositionDraft(); + } + const closeDialog = event.target.closest("[data-close-dialog]"); if (closeDialog) $(closeDialog.dataset.closeDialog)?.close(); diff --git a/src/real2sim_platform/static/assets/robots/rm65.glb b/src/real2sim_platform/static/assets/robots/rm65.glb new file mode 100644 index 0000000..b3dad1f Binary files /dev/null and b/src/real2sim_platform/static/assets/robots/rm65.glb differ diff --git a/src/real2sim_platform/static/index.html b/src/real2sim_platform/static/index.html index d4db7f0..5084ab9 100644 --- a/src/real2sim_platform/static/index.html +++ b/src/real2sim_platform/static/index.html @@ -52,6 +52,7 @@ + @@ -386,6 +387,111 @@ +
+
+
+

场景组合

+

选择基础场景与机器人,将真实数字资产拖入三维画布;保存后生成可测试、可采集的 MotrixSim 场景版本。

+
+ +
+ +
+
+
+
+

资源库

+

资源来自平台目录,拖入中间画布进行布置

+
+
+
+ +
机器人选择一个 MotrixSim 机器人
+
+
数字资产拖动或点击加入场景
+
+
+
+ +
+ +
+ +
释放以加入场景
+
+
对象层级0 个对象
+
+
+ +
+
+
+

属性检查器

+

修改选中对象的坐标与 MotrixSim 物理属性

+
+ +
+
+
选择画布或层级中的对象
+
+ 变换 +
+ + + + + + + + + +
+
+
+ 物理属性 + + + +
+
+
+
+
+ +
+ + + + +
+ +
+
+
+

组合与测试任务

+

保存、运行时校验和失败原因均显示在这里

+
+
+
+ + 已选择 0 项 + + + +
+
+
+
+
diff --git a/src/real2sim_platform/static/scene-viewer.html b/src/real2sim_platform/static/scene-viewer.html index 7fe4a4b..4698902 100644 --- a/src/real2sim_platform/static/scene-viewer.html +++ b/src/real2sim_platform/static/scene-viewer.html @@ -72,10 +72,11 @@ const params = new URLSearchParams(location.search); const sceneId = params.get("scene_id"); const embeddedMode = params.get("embedded") === "1"; + const composerMode = params.get("composer") === "1"; const coverMode = params.get("cover") === "1"; const coverView = params.get("cover_view") || "hero"; const probeFull = params.get("probe_full") === "1"; - if (coverMode) document.getElementById("hud").style.display = "none"; + if (coverMode || composerMode) document.getElementById("hud").style.display = "none"; setupHudPanel(); if (embeddedMode || innerWidth <= 640) setHudCollapsed(true); const nativeWarn = console.warn.bind(console); @@ -118,6 +119,11 @@ let loadedObject = null; let overlayObjects = []; + const draftObjects = new Map(); + let selectedDraftId = ""; + let selectionHelper = null; + let composerTool = "select"; + let compositionDrag = null; let manifestAssets = []; let manifestBounds = null; let sourceUpAxis = "Z"; @@ -136,6 +142,7 @@ }; window.real2simViewerDebug = debug; window.real2simViewerDebug.snapshot = () => ({ ...debug, canvas: { ...debug.canvas } }); + if (composerMode) setupCompositionEditor(); function setupHudPanel() { let drag = null; @@ -224,7 +231,7 @@ return; } const selected = assets[Number(button.dataset.index)]; - if (selected?.role === "added_asset") { + if (["added_asset", "robot"].includes(selected?.role)) { status.textContent = `已添加资产 · ${selected.name || "asset"} · 与 MotrixSim 场景使用同一坐标`; return; } @@ -528,7 +535,7 @@ async function loadSceneOverlays(assets) { clearOverlays(); if (!loadedObject) return; - const overlays = assets.filter((asset) => asset?.role === "added_asset"); + const overlays = assets.filter((asset) => ["added_asset", "robot"].includes(asset?.role)); for (const asset of overlays) { try { const object = await createOverlayObject(asset); @@ -548,9 +555,44 @@ } async function createOverlayObject(asset) { + if (Array.isArray(asset.viewer_components) && asset.viewer_components.length) { + const group = new THREE.Group(); + for (const component of asset.viewer_components) { + const child = createPrimitiveObject( + String(component.viewer_primitive || "box").toLowerCase(), + Array.isArray(component.dimensions) ? component.dimensions.map(Number) : [0.1, 0.1, 0.1], + Array.isArray(component.rgba) ? component.rgba.map(Number) : [0.13, 0.76, 0.67, 1], + ); + if (!child) continue; + const translation = Array.isArray(component.translation) ? component.translation.map(Number) : [0, 0, 0]; + const rotation = Array.isArray(component.rotation_euler_degrees) ? component.rotation_euler_degrees.map(Number) : [0, 0, 0]; + child.position.set(translation[0] || 0, translation[1] || 0, translation[2] || 0); + child.rotation.set(THREE.MathUtils.degToRad(rotation[0] || 0), THREE.MathUtils.degToRad(rotation[1] || 0), THREE.MathUtils.degToRad(rotation[2] || 0)); + group.add(child); + } + return group; + } const primitive = String(asset.viewer_primitive || "").toLowerCase(); const dimensions = Array.isArray(asset.physics?.dimensions) ? asset.physics.dimensions.map(Number) : [0.1, 0.1, 0.1]; const rgba = Array.isArray(asset.physics?.rgba) ? asset.physics.rgba.map(Number) : [0.13, 0.76, 0.67, 1]; + const primitiveObject = createPrimitiveObject(primitive, dimensions, rgba); + if (primitiveObject) return primitiveObject; + if (!asset.url) return null; + const suffix = String(asset.name || asset.path || asset.url || "").toLowerCase().split("?")[0].split(".").pop(); + if (suffix === "glb" || suffix === "gltf") { + const { GLTFLoader } = await import("three/addons/loaders/GLTFLoader.js"); + return (await new GLTFLoader().loadAsync(asset.url)).scene; + } + if (suffix === "obj") return new OBJLoader().loadAsync(asset.url); + if (suffix === "ply") { + const geometry = await new PLYLoader().loadAsync(asset.url); + geometry.computeVertexNormals(); + return new THREE.Mesh(geometry, material); + } + return null; + } + + function createPrimitiveObject(primitive, dimensions, rgba) { const material = new THREE.MeshStandardMaterial({ color: new THREE.Color(rgba[0], rgba[1], rgba[2]), transparent: rgba[3] < 1, @@ -580,18 +622,6 @@ group.add(first, second); return group; } - if (!asset.url) return null; - const suffix = String(asset.name || asset.path || "").toLowerCase().split(".").pop(); - if (suffix === "glb" || suffix === "gltf") { - const { GLTFLoader } = await import("three/addons/loaders/GLTFLoader.js"); - return (await new GLTFLoader().loadAsync(asset.url)).scene; - } - if (suffix === "obj") return new OBJLoader().loadAsync(asset.url); - if (suffix === "ply") { - const geometry = await new PLYLoader().loadAsync(asset.url); - geometry.computeVertexNormals(); - return new THREE.Mesh(geometry, material); - } return null; } @@ -610,6 +640,158 @@ debug.overlayCount = 0; } + function setupCompositionEditor() { + const raycaster = new THREE.Raycaster(); + const pointer = new THREE.Vector2(); + const dragPlane = new THREE.Plane(); + + addEventListener("message", (event) => { + if (event.origin !== location.origin || event.data?.type !== "real2sim-composition-draft") return; + composerTool = ["select", "move", "rotate", "scale"].includes(event.data.tool) ? event.data.tool : "select"; + loadCompositionDraft(event.data).catch((error) => console.error("Unable to load composition draft", error)); + }); + + renderer.domElement.addEventListener("pointerdown", (event) => { + const hit = compositionHit(event, raycaster, pointer); + if (!hit) return; + selectDraftObject(hit.userData.compositionInstanceId, true); + if (composerTool === "select") return; + const object = draftObjects.get(hit.userData.compositionInstanceId); + if (!object) return; + compositionDrag = { + instanceId: hit.userData.compositionInstanceId, + object, + startX: event.clientX, + startY: event.clientY, + transform: cloneTransform(object.userData.compositionTransform), + }; + if (composerTool === "move") { + contentRoot.updateMatrixWorld(true); + const localPoint = new THREE.Vector3(0, 0, Number(compositionDrag.transform.translation?.[2] || 0)).applyMatrix4(contentRoot.matrixWorld); + const normal = new THREE.Vector3(0, 0, 1).transformDirection(contentRoot.matrixWorld); + dragPlane.setFromNormalAndCoplanarPoint(normal, localPoint); + } + controls.enabled = false; + renderer.domElement.setPointerCapture(event.pointerId); + parent.postMessage({ type: "real2sim-composition-transform-start", instance_id: compositionDrag.instanceId }, location.origin); + event.preventDefault(); + }); + + renderer.domElement.addEventListener("pointermove", (event) => { + if (!compositionDrag) return; + const transform = cloneTransform(compositionDrag.transform); + if (composerTool === "move") { + setPointer(event, pointer); + raycaster.setFromCamera(pointer, camera); + const point = new THREE.Vector3(); + if (raycaster.ray.intersectPlane(dragPlane, point)) { + contentRoot.worldToLocal(point); + transform.translation[0] = point.x; + transform.translation[1] = point.y; + } + } else if (composerTool === "rotate") { + transform.rotation_euler_degrees[2] += (event.clientX - compositionDrag.startX) * 0.45; + } else if (composerTool === "scale") { + const factor = Math.max(0.05, Math.exp((compositionDrag.startY - event.clientY) * 0.01)); + transform.scale = compositionDrag.transform.scale.map((value) => Number(value) * factor); + } + compositionDrag.object.userData.compositionTransform = transform; + applyAssetTransform(compositionDrag.object, transform); + selectionHelper?.update(); + parent.postMessage({ type: "real2sim-composition-transform", instance_id: compositionDrag.instanceId, transform }, location.origin); + event.preventDefault(); + }); + + const finishDrag = (event) => { + if (!compositionDrag) return; + const instanceId = compositionDrag.instanceId; + compositionDrag = null; + controls.enabled = true; + if (renderer.domElement.hasPointerCapture?.(event.pointerId)) renderer.domElement.releasePointerCapture(event.pointerId); + parent.postMessage({ type: "real2sim-composition-transform-end", instance_id: instanceId }, location.origin); + }; + renderer.domElement.addEventListener("pointerup", finishDrag); + renderer.domElement.addEventListener("pointercancel", finishDrag); + + function compositionHit(event, raycasterInstance, pointerInstance) { + setPointer(event, pointerInstance); + raycasterInstance.setFromCamera(pointerInstance, camera); + const hits = raycasterInstance.intersectObjects([...draftObjects.values()], true); + for (const result of hits) { + let object = result.object; + while (object && !object.userData.compositionInstanceId) object = object.parent; + if (object?.userData.compositionInstanceId) return object; + } + return null; + } + + function setPointer(event, target) { + const rect = renderer.domElement.getBoundingClientRect(); + target.x = ((event.clientX - rect.left) / rect.width) * 2 - 1; + target.y = -((event.clientY - rect.top) / rect.height) * 2 + 1; + } + } + + async function loadCompositionDraft(payload) { + const requested = [payload.robot, ...(Array.isArray(payload.assets) ? payload.assets : [])].filter(Boolean); + const requestedIds = new Set(requested.map((item) => String(item.instance_id || ""))); + for (const [id, object] of draftObjects) { + if (requestedIds.has(id)) continue; + contentRoot.remove(object); + draftObjects.delete(id); + } + for (const asset of requested) { + const instanceId = String(asset.instance_id || ""); + if (!instanceId) continue; + let object = draftObjects.get(instanceId); + if (!object) { + object = await createOverlayObject(asset); + if (!object) { + debug.warnings.push(`composition draft ${asset.name || instanceId} has no browser-renderable asset`); + continue; + } + object.userData.compositionInstanceId = instanceId; + object.traverse((child) => { child.userData.compositionInstanceId = instanceId; }); + contentRoot.add(object); + draftObjects.set(instanceId, object); + } + object.userData.compositionTransform = cloneTransform(asset.transform || {}); + applyAssetTransform(object, object.userData.compositionTransform); + } + selectedDraftId = String(payload.selected_id || selectedDraftId || ""); + selectDraftObject(selectedDraftId, false); + debug.compositionDraftCount = draftObjects.size; + debug.compositionObjects = [...draftObjects.entries()].map(([instanceId, object]) => { + const bounds = new THREE.Box3().setFromObject(object); + const size = bounds.getSize(new THREE.Vector3()); + const center = bounds.getCenter(new THREE.Vector3()); + return { instanceId, size: size.toArray(), center: center.toArray() }; + }); + document.body.dataset.compositionDraftCount = String(draftObjects.size); + document.body.dataset.compositionBounds = JSON.stringify(debug.compositionObjects); + } + + function selectDraftObject(instanceId, notify) { + selectedDraftId = instanceId; + if (selectionHelper) { + scene.remove(selectionHelper); + selectionHelper = null; + } + const object = draftObjects.get(instanceId); + if (object) { + selectionHelper = new THREE.BoxHelper(object, 0x32dcc0); + scene.add(selectionHelper); + } + if (notify) parent.postMessage({ type: "real2sim-composition-selected", instance_id: instanceId }, location.origin); + } + + function cloneTransform(transform) { + const translation = Array.isArray(transform.translation) ? transform.translation.map(Number) : [0, 0, 0]; + const rotation = Array.isArray(transform.rotation_euler_degrees) ? transform.rotation_euler_degrees.map(Number) : [0, 0, 0]; + const scale = Array.isArray(transform.scale) ? transform.scale.map(Number) : [1, 1, 1]; + return { translation, rotation_euler_degrees: rotation, scale }; + } + function pointSizeFor(count, asset) { const name = String(asset?.name || asset?.path || "").toLowerCase(); if (name.includes("colmap_sparse") || name.includes("sparse_points")) return 0.022; @@ -764,6 +946,7 @@ function animate() { controls.update(); + selectionHelper?.update(); debug.canvas = { width: renderer.domElement.width, height: renderer.domElement.height }; renderer.render(scene, camera); requestAnimationFrame(animate); diff --git a/src/real2sim_platform/static/styles.css b/src/real2sim_platform/static/styles.css index 6ff1566..b52c58e 100644 --- a/src/real2sim_platform/static/styles.css +++ b/src/real2sim_platform/static/styles.css @@ -276,10 +276,62 @@ menu { display: flex; justify-content: flex-end; gap: 8px; margin: 0; padding: 1 .toast.show { opacity: 1; transform: none; } .viewer-mode [data-open-import], .viewer-mode .bulk-bar button, .viewer-mode [data-delete-dataset], .viewer-mode [data-delete-scene], .viewer-mode [data-delete-model] { display: none; } +.composition-workbench { display: grid; grid-template-columns: minmax(230px, .72fr) minmax(560px, 1.8fr) minmax(270px, .82fr); gap: 10px; align-items: stretch; } +.composition-library, .composition-stage-panel, .composition-inspector { min-height: 640px; } +.composition-library-body, .composition-inspector-body { display: grid; gap: 12px; padding-top: 12px; } +.composition-section-head, .composition-hierarchy-head { display: flex; align-items: center; justify-content: space-between; gap: 10px; color: var(--muted); font-size: 10px; } +.composition-section-head strong, .composition-hierarchy-head strong { color: var(--text); font-size: 12px; } +.composition-resource-list { display: grid; gap: 8px; } +.composition-resource-card { display: grid; grid-template-columns: 54px minmax(0, 1fr); gap: 10px; min-height: 78px; padding: 9px; border: 1px solid var(--line); border-radius: 6px; background: #091720; text-align: left; cursor: grab; } +.composition-resource-card:active { cursor: grabbing; } +.composition-resource-card.active { border-color: var(--accent); box-shadow: inset 0 0 0 1px rgba(52, 209, 187, .16); } +.composition-resource-card .resource-mark { width: 54px; height: 54px; display: grid; place-items: center; overflow: hidden; border: 1px solid var(--line-soft); border-radius: 5px; background: #061016; color: var(--accent); font-size: 10px; font-weight: 750; } +.composition-resource-card .resource-mark img { width: 100%; height: 100%; object-fit: contain; } +.composition-resource-card strong, .composition-asset-card strong { display: block; font-size: 12px; } +.composition-resource-card span, .composition-asset-card span { display: block; margin-top: 4px; color: var(--muted); font-size: 9px; line-height: 1.35; } +.composition-asset-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 7px; max-height: 388px; overflow: auto; padding-right: 2px; } +.composition-asset-card { min-height: 88px; display: grid; align-content: space-between; gap: 8px; padding: 9px; border: 1px solid var(--line); border-radius: 6px; background: #091720; color: var(--text); text-align: left; cursor: grab; } +.composition-asset-card:hover, .composition-resource-card:hover { border-color: #39786f; background: #0b1c25; } +.composition-asset-card[data-in-scene="true"] { border-color: #2b665e; background: #0b211f; } +.composition-asset-card em { color: var(--accent); font-size: 9px; font-style: normal; } +.composition-toolbar { min-height: 48px; display: flex; align-items: center; gap: 5px; padding: 8px 11px; border-bottom: 1px solid var(--line-soft); } +.composition-toolbar button { min-height: 30px; padding: 5px 9px; } +.composition-toolbar button.active { border-color: var(--accent); background: #103b38; color: #eafffb; } +.toolbar-separator { width: 1px; height: 22px; margin: 0 4px; background: var(--line); } +.composition-stage-status { min-width: 0; margin-left: auto; color: var(--muted); font-size: 10px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.composition-canvas-wrap { position: relative; height: 476px; margin: 0 !important; overflow: hidden; background: #030709; } +.composition-canvas-wrap iframe { width: 100%; height: 100%; border: 0; background: #030709; } +.composition-drop-overlay { position: absolute; inset: 10px; z-index: 4; display: none; place-items: center; border: 1px dashed var(--accent); border-radius: 6px; background: rgba(7, 31, 31, .82); color: #d9fff9; font-size: 14px; font-weight: 700; pointer-events: none; } +body.composition-dragging .composition-drop-overlay { display: grid; pointer-events: auto; } +.composition-hierarchy-head { min-height: 34px; padding: 7px 11px; border-top: 1px solid var(--line-soft); border-bottom: 1px solid var(--line-soft); } +.composition-hierarchy { max-height: 120px; overflow: auto; margin: 0 !important; } +.composition-tree-row { min-height: 31px; display: grid; grid-template-columns: minmax(0, 1fr) 92px 42px; align-items: center; gap: 7px; padding: 5px 11px; border-bottom: 1px solid var(--line-soft); cursor: pointer; } +.composition-tree-row:hover, .composition-tree-row.active { background: #0d222b; } +.composition-tree-row.active { box-shadow: inset 2px 0 var(--accent); } +.composition-tree-row span { color: var(--muted); font-size: 9px; } +.composition-tree-row button { min-height: 24px; padding: 2px 6px; font-size: 9px; } +.composition-inspector fieldset { display: grid; gap: 9px; margin: 0; padding: 10px; border: 1px solid var(--line-soft); border-radius: 6px; } +.composition-inspector legend { padding: 0 5px; color: var(--muted); font-size: 10px; } +.composition-selection-card { min-height: 54px; padding: 9px; border: 1px solid var(--line); border-radius: 6px; background: #08141c; } +.composition-selection-card strong { display: block; } +.composition-selection-card span { display: block; margin-top: 4px; color: var(--muted); font-size: 9px; } +.axis-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 7px; } +.axis-grid label { min-width: 0; font-size: 9px; } +.axis-grid input { min-width: 0; padding: 7px 6px; } +.compatibility-card { display: grid; gap: 6px; padding: 10px; border: 1px solid #255f57; border-radius: 6px; background: #09201e; color: var(--muted); font-size: 10px; line-height: 1.45; } +.compatibility-card strong { color: #74dfcb; } +.composition-action-bar { position: sticky; bottom: 8px; z-index: 20; display: grid; grid-template-columns: minmax(240px, 1fr) minmax(180px, .75fr) minmax(180px, .75fr) minmax(180px, .75fr); align-items: end; gap: 9px; margin-top: 10px; padding: 10px 12px; border: 1px solid var(--line); border-radius: 7px; background: rgba(8, 20, 27, .96); box-shadow: 0 14px 40px rgba(0, 0, 0, .38); backdrop-filter: blur(8px); } +.composition-action-bar button { min-height: 39px; } +.composition-tasks-panel { margin-top: 10px; } +.composition-inspector-body > fieldset[disabled] { opacity: .45; } + @media (max-width: 1180px) { - .overview-grid, .two-column, .reconstruct-grid, .scene-workbench { grid-template-columns: 1fr; } + .overview-grid, .two-column, .reconstruct-grid, .scene-workbench, .composition-workbench { grid-template-columns: 1fr; } .overview-grid .wide { grid-column: auto; } .scene-workbench .viewer-panel { order: -1; } + .composition-stage-panel { order: -1; } + .composition-library, .composition-stage-panel, .composition-inspector { min-height: 0; } + .composition-action-bar { grid-template-columns: repeat(2, minmax(0, 1fr)); } .asset-grid, .model-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .selection-card-rail { grid-template-columns: repeat(3, minmax(0, 1fr)); } } @@ -319,6 +371,12 @@ menu { display: flex; justify-content: flex-end; gap: 8px; margin: 0; padding: 1 .viewer-panel iframe, #scenePreview { width: calc(100% - 26px); min-height: 390px; } .task-facts { grid-template-columns: 1fr; } .task-drawer { width: 100vw; } + .composition-canvas-wrap { height: 420px; } + .composition-action-bar { position: static; grid-template-columns: 1fr; } + .composition-asset-grid { display: flex; max-height: none; overflow-x: auto; } + .composition-asset-card { flex: 0 0 140px; } + .composition-toolbar { overflow-x: auto; } + .composition-stage-status { display: none; } .toolbar { width: 100%; } .toolbar input, .toolbar select { width: 100%; } .user-row { align-items: start; flex-direction: column; } diff --git a/tests/test_production_platform.py b/tests/test_production_platform.py index 5a4260c..527d82f 100644 --- a/tests/test_production_platform.py +++ b/tests/test_production_platform.py @@ -20,6 +20,8 @@ from real2sim_platform.adapters import ( index_artifacts_for_task, ) from real2sim_platform.motrix_scene import ( + COMPOSITION_ASSET_RECIPES, + build_scene_composition, compose_scene_asset, convert_asset_to_obj, detect_spatial_bundle, @@ -206,6 +208,104 @@ def test_scene_revision_copy_is_isolated_and_rebases_package_paths(tmp_path: Pat assert (source / "scene_manifest.json").read_bytes() == source_manifest_before +def test_scene_composition_catalog_and_payload_are_registered(tmp_path: Path) -> None: + config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api") + app = create_app(config) + package = tmp_path / "editable-scene" + runtime = package / "simulator_assets" / "scene.xml" + runtime.parent.mkdir(parents=True) + runtime.write_text('\n', encoding="utf-8") + manifest = package / "scene_manifest.json" + manifest.write_text(json.dumps({"assets": [], "simulator_artifacts": {"motrixsim": {"runtime_scene": str(runtime)}}}), encoding="utf-8") + scene = PlatformStore(config.db_path).create_scene( + name="editable", + path=runtime, + metadata={"package": str(package), "editable": True}, + source_type="motrixsim_spatial_bundle", + manifest_path=manifest, + ) + with TestClient(app) as client: + robots = client.get("/api/robots") + assert robots.status_code == 200 + assert robots.json()[0]["id"] == "rm65" + assets = client.get("/api/assets").json() + assert {"recipe:coffee_table", "recipe:pick_cube", "recipe:place_target"}.issubset({item["id"] for item in assets}) + invalid = client.post( + f"/api/scenes/{scene['id']}/compose", + json={"name": "invalid", "robot": {"robot_id": "rm65"}, "assets": [{"catalog_id": "missing", "instance_id": "missing"}]}, + ) + assert invalid.status_code == 400 + queued = client.post( + f"/api/scenes/{scene['id']}/compose", + json={ + "name": "RM65 pick place", + "robot": {"robot_id": "rm65", "transform": {"translation": [1.03, 0, 0.011], "scale": [1, 1, 1]}}, + "assets": [ + {"catalog_id": recipe_id, "instance_id": f"asset_{index}", "transform": COMPOSITION_ASSET_RECIPES[recipe_id]["default_transform"]} + for index, recipe_id in enumerate(("recipe:coffee_table", "recipe:pick_cube", "recipe:place_target")) + ], + "validation_steps": 120, + }, + ) + assert queued.status_code == 200, queued.text + assert queued.json()["task"]["type"] == "scene_composition" + + +@pytest.mark.integration +def test_real_rm65_scene_composition_loads_and_steps_in_both_runtimes(tmp_path: Path) -> None: + mujoco = pytest.importorskip("mujoco") + pytest.importorskip("motrixsim") + candidates = [ + Path("/opt/motrixsim-docs/examples/assets/rm65_open_cabinet/mjx_rm65.xml"), + Path("/Users/d-robotics/workSpace/motrixsim-docs/examples/assets/rm65_open_cabinet/mjx_rm65.xml"), + ] + robot_source = next((path for path in candidates if path.is_file()), None) + if robot_source is None: + pytest.skip("real MotrixSim RM65 asset checkout is unavailable") + package = tmp_path / "composition" + runtime = package / "simulator_assets" / "scene.xml" + (package / "assets" / "meshes").mkdir(parents=True) + runtime.parent.mkdir(parents=True) + runtime.write_text( + '\n', + encoding="utf-8", + ) + (package / "scene_manifest.json").write_text(json.dumps({"assets": [], "simulator_artifacts": {"motrixsim": {"runtime_scene": str(runtime)}}}), encoding="utf-8") + (package / "motrix_import.json").write_text(json.dumps({"runtime_scene": str(runtime)}), encoding="utf-8") + asset_specs = [ + { + "catalog_id": recipe_id, + "instance_id": f"asset_{index}", + "name": COMPOSITION_ASSET_RECIPES[recipe_id]["name"], + "transform": COMPOSITION_ASSET_RECIPES[recipe_id]["default_transform"], + } + for index, recipe_id in enumerate(("recipe:coffee_table", "recipe:pick_cube", "recipe:place_target")) + ] + result = build_scene_composition( + package, + robot_source=robot_source, + robot_spec={"robot_id": "rm65", "transform": {"translation": [1.03, 0, 0.011], "rotation_euler_degrees": [0, 0, 0], "scale": [1, 1, 1]}}, + asset_specs=asset_specs, + source_paths={}, + validation_steps=120, + ) + assert result["validation"]["mujoco"]["status"] == "loaded_and_stepped" + assert result["validation"]["motrixsim"]["status"] == "loaded_and_stepped" + assert result["collection_ready"] is True + model = mujoco.MjModel.from_xml_path(str(runtime)) + assert model.nkey >= 1 + for object_type, name in ( + (mujoco.mjtObj.mjOBJ_BODY, "base_link"), + (mujoco.mjtObj.mjOBJ_BODY, "pick_cube"), + (mujoco.mjtObj.mjOBJ_BODY, "place_target"), + (mujoco.mjtObj.mjOBJ_JOINT, "pick_cube_free"), + (mujoco.mjtObj.mjOBJ_GEOM, "pick_cube_geom"), + (mujoco.mjtObj.mjOBJ_CAMERA, "overview"), + ): + assert mujoco.mj_name2id(model, object_type, name) >= 0 + assert Path(result["composition"]["robot"]["path"]).stat().st_size > 1_000_000 + + def test_ascii_3dgs_spz_exporter_uses_real_spark_writer(tmp_path: Path) -> None: config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace") probe = _spz_exporter_probe(config)