diff --git a/docker/worker.Dockerfile b/docker/worker.Dockerfile index ae2dd40..d410d8c 100644 --- a/docker/worker.Dockerfile +++ b/docker/worker.Dockerfile @@ -96,7 +96,7 @@ COPY scripts ./scripts COPY vendor/gs_playground /opt/gs_playground RUN python -m pip install --no-cache-dir --no-deps . -RUN python -c 'import mujoco, motrixsim, torch; from huggingface_hub import snapshot_download; assert torch.__version__.startswith("2.2")' && \ +RUN python -c 'import mujoco, motrixsim, torch, trimesh; from pxr import Usd, UsdGeom; from huggingface_hub import snapshot_download; assert torch.__version__.startswith("2.2"); assert Usd is not None and UsdGeom is not None and trimesh is not None' && \ test "$(node --version)" = "v20.19.2" && \ PYTHONPATH=/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE python -c 'from policies.act.act import ACTPolicy; assert ACTPolicy is not None' && \ PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat python -c 'from src.model.model.anysplat import AnySplat; assert AnySplat is not None' && \ diff --git a/pyproject.toml b/pyproject.toml index 2baae60..04b16cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,11 @@ dependencies = [ "pillow>=10", "pyarrow>=15", "mujoco>=3.2", + "numpy>=1.25,<2", "pydantic>=2", "python-multipart>=0.0.9", + "trimesh>=4.12,<5", + "usd-core==25.11", "uvicorn[standard]>=0.27", ] diff --git a/scripts/acceptance_spatial_bundle_poc.py b/scripts/acceptance_spatial_bundle_poc.py new file mode 100644 index 0000000..2f9e1e1 --- /dev/null +++ b/scripts/acceptance_spatial_bundle_poc.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import os +import time +from pathlib import Path +from typing import Any + +from acceptance_poc_cluster import ( + DEFAULT_KUBECONFIG, + NAMESPACE, + AcceptanceError, + AcceptanceRun, + ApiClient, + authenticate_admin, + finalize, + free_port, + kubectl, + start_port_forward, + upload_with_resume, +) + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BUNDLE = Path("/Users/d-robotics/Downloads/demo_2.zip") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def dataset_for_checksum(run: AcceptanceRun, checksum: str) -> dict[str, Any] | None: + for dataset in run.client.get("/api/datasets"): + if str(dataset.get("checksum") or "").lower() == checksum.lower(): + return dataset + return None + + +def verify_spatial_dataset(run: AcceptanceRun, dataset: dict[str, Any], expected_checksum: str) -> None: + scan = dataset.get("scan") or {} + run.check("空间包 SHA-256 与原文件一致", dataset.get("checksum") == expected_checksum, dataset.get("checksum")) + run.check("识别为 MotrixSim 空间包", scan.get("input_kind") == "motrixsim_spatial_bundle", scan) + run.check("USDZ 碰撞、视觉和 3DGS 三层完整", bool(scan.get("spatial_bundle_ready")), scan) + run.check("原始空间包已持久化到受控 BOS 前缀", str(dataset.get("storage_uri") or "").startswith("bos://testxyz/real2sim-poc/uploads/"), dataset.get("storage_uri")) + + +def import_scene(run: AcceptanceRun, dataset_id: str) -> dict[str, Any]: + created = run.client.post(f"/api/datasets/{dataset_id}/import-motrixsim") + task = run.wait_task( + created["id"], + required_artifacts=( + "scene_manifest.json", + "motrixsim_scene_import_report.json", + "runtime_validation.json", + "scene.xml", + "environment_visual.glb", + "full_res.spz", + ), + timeout_seconds=7200, + ) + result = task.get("result") or {} + runtime = result.get("runtime_validation") or {} + run.check("MuJoCo 真实加载并步进导入场景", bool((runtime.get("mujoco") or {}).get("ok")), runtime) + run.check("MotrixSim 真实加载并步进导入场景", bool((runtime.get("motrixsim") or {}).get("ok")), runtime) + scene_id = str(result.get("scene_id") or "") + run.check("空间包已登记为平台场景", bool(scene_id), result) + manifest = run.client.get(f"/api/scenes/{scene_id}/manifest") + formats = {str(item.get("format") or "").lower() for item in manifest.get("assets", [])} + names = {str(item.get("name") or "") for item in manifest.get("assets", [])} + run.check("场景保留 SPZ 高保真外观层", "spz" in formats or "full_res.spz" in names, names) + run.check("场景生成可交互 GLB 回退预览", "glb" in formats or "environment_visual.glb" in names, names) + run.report["scenes"].append({"id": scene_id, "role": "spatial_bundle_import", "manifest": manifest}) + return {"scene_id": scene_id, "manifest": manifest, "task": task} + + +def add_asset_revision( + run: AcceptanceRun, + *, + scene_id: str, + primitive: str, + name: str, + translation: list[float], + dimensions: list[float], + dynamic: bool, +) -> dict[str, Any]: + created = run.client.post( + f"/api/scenes/{scene_id}/assets", + { + "source": {"type": "primitive"}, + "primitive": primitive, + "name": name, + "transform": { + "translation": translation, + "rotation_euler_degrees": [0.0, 0.0, 0.0], + "scale": [1.0, 1.0, 1.0], + }, + "physics": { + "dynamic": dynamic, + "mass": 0.35 if dynamic else 1.0, + "collision_mode": "visual_and_collision", + "dimensions": dimensions, + "friction": [0.8, 0.01, 0.001], + "rgba": [0.12, 0.78, 0.68, 1.0], + }, + "validation_steps": 120, + "revision_name": f"空间场景 · {name}", + }, + ) + task_id = str((created.get("task") or {}).get("id") or "") + run.check(f"{name} 合成任务已创建", bool(task_id), created) + task = run.wait_task( + task_id, + required_artifacts=("scene_manifest.json", "scene.xml"), + timeout_seconds=7200, + ) + result = task.get("result") or {} + runtime = result.get("runtime_validation") or {} + run.check(f"{name} 修订通过 MuJoCo", bool((runtime.get("mujoco") or {}).get("ok")), runtime) + run.check(f"{name} 修订通过 MotrixSim", bool((runtime.get("motrixsim") or {}).get("ok")), runtime) + if dynamic: + run.check(f"{name} 在 MotrixSim 中具有真实自由关节", int((runtime.get("motrixsim") or {}).get("nq") or 0) >= 7, runtime) + revision_id = str(result.get("scene_id") or "") + run.check(f"{name} 生成独立场景版本", bool(revision_id) and revision_id != scene_id, result) + manifest = run.client.get(f"/api/scenes/{revision_id}/manifest") + added = [item for item in manifest.get("assets", []) if item.get("role") == "added_asset"] + run.check(f"{name} 已写入场景清单", any(item.get("name") == name for item in added), added) + run.report["scenes"].append( + { + "id": revision_id, + "role": f"asset_revision_{primitive}", + "parent_scene_id": scene_id, + "added_assets": [item.get("name") for item in added], + "runtime_validation": runtime, + } + ) + return {"scene_id": revision_id, "manifest": manifest, "task": task} + + +def run_acceptance(args: argparse.Namespace) -> int: + bundle = args.bundle.expanduser().resolve() + if not bundle.is_file(): + raise AcceptanceError(f"spatial bundle does not exist: {bundle}") + checksum = sha256_file(bundle) + build_id = args.build_id or kubectl( + args.kubeconfig, + "-n", + NAMESPACE, + "get", + "deployment/real2sim-api", + "-o", + "jsonpath={.metadata.labels.app\\.kubernetes\\.io/version}", + ).stdout.strip() + if not build_id: + raise AcceptanceError("unable to determine deployed build id") + timestamp = time.strftime("%Y%m%d-%H%M%S") + out_dir = args.out or ROOT / "acceptance_runs" / "poc" / f"{build_id}-spatial-{timestamp}" + out_dir.mkdir(parents=True, exist_ok=True) + port = args.port or free_port() + forward = start_port_forward(args.kubeconfig, port, out_dir) + run = AcceptanceRun( + client=ApiClient(f"https://127.0.0.1:{port}"), + out_dir=out_dir, + kubeconfig=args.kubeconfig, + build_id=build_id, + timeout_seconds=args.timeout_seconds, + ) + run.report["schema"] = "real2sim.spatial_bundle_acceptance.v1" + run.report["source"] = { + "path": str(bundle), + "filename": bundle.name, + "size_bytes": bundle.stat().st_size, + "sha256": checksum, + } + try: + if run.stage("管理员认证", lambda: authenticate_admin(run)) is None: + return finalize(run) + version = run.client.get("/api/version") + doctor = run.client.get("/api/doctor") + run.report["version"] = version + run.report["doctor"] = doctor + run.check("部署版本与验收目标一致", version.get("build_id") == build_id, version) + run.check("生产运行时 doctor 通过", bool(doctor.get("ok")), doctor) + + dataset = dataset_for_checksum(run, checksum) if args.reuse else None + if dataset is None: + dataset = run.stage("真实空间包分片上传", lambda: upload_with_resume(run, bundle, checksum)) + else: + run.report["warnings"].append(f"Reused dataset {dataset['id']} with identical SHA-256.") + if dataset is None: + return finalize(run) + verify_spatial_dataset(run, dataset, checksum) + + imported = run.stage("USDZ/SPZ 转 MotrixSim 场景", lambda: import_scene(run, dataset["id"])) + if not imported: + return finalize(run) + cube = run.stage( + "添加动态抓取方块并生成修订", + lambda: add_asset_revision( + run, + scene_id=imported["scene_id"], + primitive="box", + name="动态抓取方块", + translation=[0.0, 0.0, 1.0], + dimensions=[0.6, 0.6, 0.6], + dynamic=True, + ), + ) + if not cube: + return finalize(run) + marker = run.stage( + "添加静态目标圆柱并保留上一资产", + lambda: add_asset_revision( + run, + scene_id=cube["scene_id"], + primitive="cylinder", + name="静态放置目标", + translation=[1.2, 0.0, 0.6], + dimensions=[0.8, 0.8, 1.2], + dynamic=False, + ), + ) + if marker: + added_names = { + str(item.get("name") or "") + for item in marker["manifest"].get("assets", []) + if item.get("role") == "added_asset" + } + run.check("连续场景修订保留全部数字资产", {"动态抓取方块", "静态放置目标"}.issubset(added_names), sorted(added_names)) + run.report["browser"]["target_scene_id"] = marker["scene_id"] + run.report["browser"]["viewer_url"] = f"/static/scene-viewer.html?scene_id={marker['scene_id']}" + return finalize(run) + finally: + forward.terminate() + try: + forward.wait(timeout=10) + except Exception: + forward.kill() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import a real USDZ/SPZ spatial bundle, validate MotrixSim, and compose digital assets in the isolated POC namespace.") + parser.add_argument("--bundle", type=Path, default=Path(os.environ.get("REAL2SIM_SPATIAL_BUNDLE", DEFAULT_BUNDLE))) + parser.add_argument("--kubeconfig", type=Path, default=Path(os.environ.get("REAL2SIM_POC_KUBECONFIG", DEFAULT_KUBECONFIG))) + parser.add_argument("--build-id", default=os.environ.get("REAL2SIM_POC_BUILD_ID", "")) + parser.add_argument("--out", type=Path) + parser.add_argument("--port", type=int, default=0) + parser.add_argument("--timeout-seconds", type=int, default=7200) + parser.add_argument("--no-reuse", dest="reuse", action="store_false") + parser.set_defaults(reuse=True) + return parser.parse_args() + + +if __name__ == "__main__": + try: + raise SystemExit(run_acceptance(parse_args())) + except KeyboardInterrupt: + raise SystemExit(130) + except Exception as exc: + print(f"spatial bundle acceptance failed: {type(exc).__name__}: {exc}", flush=True) + raise SystemExit(2) diff --git a/src/real2sim_platform/adapters.py b/src/real2sim_platform/adapters.py index 5cca638..40823c3 100644 --- a/src/real2sim_platform/adapters.py +++ b/src/real2sim_platform/adapters.py @@ -16,7 +16,8 @@ from typing import Any, Callable from urllib.parse import quote, unquote, urlparse from urllib.request import urlopen -from .config import PlatformConfig +from .config import PlatformConfig, is_path_allowed +from .motrix_scene import build_spatial_scene_package, compose_scene_asset from .source_revision import read_source_revision from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path from .store import PlatformStore @@ -25,7 +26,7 @@ from .store import PlatformStore IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".ppm"} VIDEO_SUFFIXES = {".mp4", ".mov", ".m4v"} PARQUET_SUFFIXES = {".parquet"} -SCENE_ASSET_SUFFIXES = {".ply", ".splat", ".spz", ".ksplat", ".sog", ".zip", ".rad", ".obj", ".glb", ".gltf", ".xml", ".json", ".png", ".jpg", ".jpeg"} +SCENE_ASSET_SUFFIXES = {".ply", ".splat", ".spz", ".ksplat", ".sog", ".zip", ".rad", ".obj", ".stl", ".glb", ".gltf", ".usd", ".usda", ".usdc", ".usdz", ".xml", ".json", ".png", ".jpg", ".jpeg"} SPARK_ASSET_SUFFIXES = {".ply", ".splat", ".spz", ".ksplat", ".sog", ".zip", ".rad"} SPARK_VENDOR_PATH = Path(__file__).parent / "static" / "vendor" / "spark.module.min.js" THREE_VENDOR_PATH = Path(__file__).parent / "static" / "vendor" / "three" / "three.module.js" @@ -50,14 +51,27 @@ def scan_dataset(path: Path) -> dict[str, Any]: videos = [p for p in files if p.suffix.lower() in VIDEO_SUFFIXES] parquets = [p for p in files if p.suffix.lower() in PARQUET_SUFFIXES] metadata = [p for p in files if p.name in {"metadata.json", "transforms.json", "calibration.json"} or p.suffix.lower() == ".jsonl"] + scene_assets = [p for p in files if p.suffix.lower() in SCENE_ASSET_SUFFIXES] + usdz_files = [p for p in files if p.suffix.lower() == ".usdz"] + splat_files = [p for p in files if p.suffix.lower() in {".spz", ".splat", ".ksplat", ".sog"}] + spatial_bundle = bool( + any("collider" in p.stem.lower() for p in usdz_files) + and any("shaded" in p.stem.lower() or "scene" in p.stem.lower() for p in usdz_files) + and splat_files + ) prepared_manifest = root / "prepared_manifest.json" if root.is_dir() else None - input_kind = "prepared_frames" if prepared_manifest and prepared_manifest.is_file() else "video" if videos or root.suffix.lower() in VIDEO_SUFFIXES else "parquet_video" if parquets else "rgbd" if depths else "images" + input_kind = "motrixsim_spatial_bundle" if spatial_bundle else "prepared_frames" if prepared_manifest and prepared_manifest.is_file() else "video" if videos or root.suffix.lower() in VIDEO_SUFFIXES else "parquet_video" if parquets else "rgbd" if depths else "images" return { "path": str(root), "image_count": len(images), "depth_count": len(depths), "video_count": len(videos), "parquet_count": len(parquets), + "scene_asset_count": len(scene_assets), + "usdz_count": len(usdz_files), + "splat_count": len(splat_files), + "spatial_bundle_ready": spatial_bundle, + "scene_asset_files": [str(p.relative_to(root)) if root.is_dir() else p.name for p in scene_assets[:100]], "metadata_files": [str(p.relative_to(root)) if root.is_dir() else p.name for p in metadata[:20]], "prepared_manifest": str(prepared_manifest) if prepared_manifest and prepared_manifest.is_file() else "", "input_kind": input_kind, @@ -79,6 +93,7 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]: "anysplat_pinned", "mujoco_python", "motrixsim_python", + "usd_scene_import", "nvidia_smi", ] if config.production_mode @@ -111,6 +126,7 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]: "act_import": _act_runtime_import_ready(str(config.discoverse_root), config.discoverse_commit, config.act_commit), "anysplat_pinned": _git_head(config.anysplat_root) == config.anysplat_commit and config.anysplat_python.exists(), "motrixsim_python": _module_available("motrixsim"), + "usd_scene_import": all(_module_available(name) for name in ["pxr", "trimesh", "numpy"]), } return { "ok": all(bool(checks[name]) for name in core_checks), @@ -174,6 +190,10 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A return run_bos_import(config, store, task, emit) if task_type == "motrixsim_asset_import": return run_motrixsim_asset_import(config, store, task, emit) + if task_type == "motrixsim_scene_import": + return run_motrixsim_scene_import(config, store, task, emit) + if task_type == "scene_asset_compose": + return run_scene_asset_compose(config, store, task, emit) if task_type == "simulator_validation": return run_simulator_validation(config, store, task, emit) if task_type == "expert_collection": @@ -725,6 +745,141 @@ def run_motrixsim_asset_import(config: PlatformConfig, store: PlatformStore, tas } +def run_motrixsim_scene_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: + params = task["params"] + dataset = store.get_dataset(str(params["dataset_id"])) + input_root = Path(dataset["path"]) + run_dir = Path(task["run_dir"]) + package_dir = run_dir / "scene_package" + scene_name = str(params.get("scene_name") or f"{dataset['name']} · MotrixSim 场景") + validation_steps = int(params.get("validation_steps", 120)) + + _task_progress(store, task, 15, "识别空间包", "正在校验 USDZ 碰撞层、视觉层与 3DGS 外观层") + result = build_spatial_scene_package( + input_root, + package_dir, + scene_name=scene_name, + validation_steps=validation_steps, + emit=emit, + ) + _task_progress(store, task, 84, "登记场景", "MotrixSim 已真实加载并步进,正在登记场景资产与清单") + artifacts = _index_artifacts(store, task["id"], package_dir) + manifest_path = Path(result["manifest_path"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + _attach_artifact_urls(manifest, artifacts) + manifest_path.write_text(json.dumps(enrich_scene_manifest_assets(manifest), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + report = result["report"] + validation = report["runtime_validation"] + scene = store.create_scene( + name=scene_name, + path=Path(result["runtime_scene"]), + metadata={ + "package": str(package_dir), + "renderer": "spark", + "simulators": ["mujoco", "motrixsim"], + "runtime_validation": {"mujoco": bool(validation["mujoco"]["ok"]), "motrixsim": bool(validation["motrixsim"]["ok"])}, + "source_dataset_id": dataset["id"], + "editable": True, + }, + source_type="motrixsim_spatial_bundle", + source_uri=dataset.get("storage_uri") or dataset.get("source_uri") or str(input_root), + version=str(dataset.get("version") or dataset.get("checksum") or "v1")[:24], + parent_task_id=task["id"], + manifest_path=manifest_path, + metrics=manifest.get("metrics") or {}, + tags=["motrixsim", "spatial-bundle", "usdz", "3dgs", "runtime-validated", "editable"], + cache_state="local", + ) + _task_progress(store, task, 94, "场景可用", "空间包已成为可预览、可添加数字资产的 MotrixSim 场景") + return { + "scene_id": scene["id"], + "scene_path": scene["path"], + "manifest_path": str(manifest_path), + "package_dir": str(package_dir), + "runtime_validation": validation, + "source_dataset_id": dataset["id"], + } + + +def run_scene_asset_compose(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: + params = task["params"] + parent = store.get_scene(str(params["scene_id"])) + source_package = Path(str((parent.get("metadata") or {}).get("package") or "")) + if not source_package.is_dir(): + scene_path = Path(parent["path"]) + source_package = scene_path if scene_path.is_dir() else scene_path.parent + if not (source_package / "scene_manifest.json").is_file(): + raise ValueError("selected scene is not an editable MotrixSim scene package") + + run_dir = Path(task["run_dir"]) + revision_dir = run_dir / "scene_revision" + _task_progress(store, task, 18, "创建场景版本", "正在复制父场景并保留原版本不变") + _copy_tree_hardlink(source_package, revision_dir) + source_path = _resolve_asset_source(config, store, params.get("source") or {}, params.get("path")) + asset_spec = { + "name": params.get("name"), + "primitive": params.get("primitive"), + "transform": params.get("transform") or {}, + "physics": params.get("physics") or {}, + } + _task_progress(store, task, 38, "合成数字资产", "正在把资产写入 MotrixSim MJCF 和浏览器场景层") + composed = compose_scene_asset( + revision_dir, + asset_spec=asset_spec, + source_path=source_path, + validation_steps=int(params.get("validation_steps", 60)), + emit=emit, + ) + _task_progress(store, task, 80, "登记修订产物", "双运行时校验通过,正在生成资产 URL、血缘和新场景版本") + artifacts = _index_artifacts(store, task["id"], revision_dir) + manifest_path = Path(composed["manifest_path"]) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + revision_id = uuid.uuid4().hex + manifest["revision"] = { + "id": revision_id, + "parent_scene_id": parent["id"], + "reason": "digital_asset_composed", + "task_id": task["id"], + } + _attach_artifact_urls(manifest, artifacts) + manifest_path.write_text(json.dumps(enrich_scene_manifest_assets(manifest), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + asset_name = str(composed["asset"].get("name") or "数字资产") + validation = composed["validation"] + metadata = dict(parent.get("metadata") or {}) + metadata.update( + { + "package": str(revision_dir), + "parent_scene_id": parent["id"], + "revision_reason": "digital_asset_composed", + "runtime_validation": {"mujoco": bool(validation["mujoco"]["ok"]), "motrixsim": bool(validation["motrixsim"]["ok"])}, + "editable": True, + } + ) + scene = store.create_scene( + item_id=revision_id, + name=str(params.get("revision_name") or f"{parent['name']} · {asset_name}"), + path=Path(composed["runtime_scene"]), + metadata=metadata, + source_type="scene_revision", + source_uri=parent.get("storage_uri") or parent.get("source_uri") or parent["path"], + version=_next_scene_revision(str(parent.get("version") or "v1")), + parent_task_id=task["id"], + manifest_path=manifest_path, + metrics={**(parent.get("metrics") or {}), "added_asset_count": len([item for item in manifest.get("assets", []) if item.get("role") == "added_asset"])}, + tags=list(dict.fromkeys([*(parent.get("tags") or []), "scene-revision", "digital-asset", "runtime-validated"])), + cache_state="local", + ) + _task_progress(store, task, 94, "新版本可用", "数字资产已进入真实 MotrixSim 场景,新版本可预览和继续编辑") + return { + "scene_id": scene["id"], + "parent_scene_id": parent["id"], + "scene_path": scene["path"], + "manifest_path": str(manifest_path), + "asset": composed["asset"], + "runtime_validation": validation, + } + + def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: params = task["params"] run_dir = Path(task["run_dir"]) @@ -2832,18 +2987,87 @@ def _find_scene_json(package_dir: Path) -> Path | None: return matches[0] if matches else None -def _index_artifacts(store: PlatformStore, task_id: str, root: Path) -> None: +def _copy_tree_hardlink(source: Path, target: Path) -> None: + def link_or_copy(src: str, dst: str) -> str: + try: + os.link(src, dst) + return dst + except OSError: + return shutil.copy2(src, dst) + + shutil.copytree(source, target, copy_function=link_or_copy) + + +def _resolve_asset_source(config: PlatformConfig, store: PlatformStore, source: dict[str, Any], legacy_path: Any) -> Path | None: + source_type = str(source.get("type") or "") + if source_type == "primitive" or (not source_type and not legacy_path): + return None + if source_type == "dataset": + dataset = store.get_dataset(str(source.get("dataset_id") or "")) + root = Path(dataset["path"]).resolve() + relative = Path(str(source.get("relative_path") or "")) + candidate = (root / relative).resolve() + if candidate != root and root not in candidate.parents: + raise ValueError("asset path escapes the selected dataset") + elif source_type == "scene": + scene = store.get_scene(str(source.get("scene_id") or "")) + manifest_path = Path(str(scene.get("manifest_path") or "")) + manifest = _read_optional_json(manifest_path) + key = str(source.get("asset_key") or source.get("relative_path") or "") + selected = next( + ( + item + for item in manifest.get("assets", []) + if isinstance(item, dict) and key in {str(item.get("name") or ""), str(item.get("path") or "")} + ), + None, + ) + if selected is None: + raise ValueError("selected scene asset no longer exists") + candidate = Path(str(selected.get("path") or "")).resolve() + else: + candidate = Path(str(legacy_path or source.get("path") or "")).expanduser().resolve() + if not is_path_allowed(candidate, config): + raise ValueError("asset path is outside configured import roots") + if not candidate.is_file(): + raise FileNotFoundError(candidate) + return candidate + + +def _attach_artifact_urls(manifest: dict[str, Any], artifacts: list[dict[str, Any]]) -> None: + by_path = {str(Path(item["path"]).resolve()): item for item in artifacts} + for asset in manifest.get("assets", []): + if not isinstance(asset, dict) or not asset.get("path"): + continue + artifact = by_path.get(str(Path(str(asset["path"])).resolve())) + if artifact: + asset["url"] = f"/api/artifacts/{artifact['id']}/content" + + +def _next_scene_revision(current: str) -> str: + import re + + match = re.search(r"(?:^|[-_.])r(\d+)$", current, re.IGNORECASE) + number = int(match.group(1)) + 1 if match else 2 + base = re.sub(r"(?:[-_.])r\d+$", "", current, flags=re.IGNORECASE).strip("-_.") or "v1" + return f"{base}-r{number}" + + +def _index_artifacts(store: PlatformStore, task_id: str, root: Path) -> list[dict[str, Any]]: indexable_suffixes = { ".json", ".jsonl", ".html", ".svg", ".png", ".jpg", ".jpeg", ".mp4", ".zip", ".tar", ".gz", - ".ply", ".spz", ".splat", ".ksplat", ".sog", ".rad", ".obj", ".glb", ".gltf", ".xml", + ".ply", ".spz", ".splat", ".ksplat", ".sog", ".rad", ".obj", ".stl", ".glb", ".gltf", + ".usd", ".usda", ".usdc", ".usdz", ".xml", ".hdf5", ".h5", ".ckpt", ".pkl", ".npy", ".npz", ".csv", ".yaml", ".yml", ".toml", ".py", ".log", ".txt", } + artifacts: list[dict[str, Any]] = [] for path in sorted(root.rglob("*")): if not path.is_file(): continue if path.suffix.lower() in indexable_suffixes: - store.add_artifact(task_id=task_id, name=path.name, path=path, kind=path.suffix.lower().lstrip(".") or "file") + artifacts.append(store.add_artifact(task_id=task_id, name=path.name, path=path, kind=path.suffix.lower().lstrip(".") or "file")) + return artifacts def index_artifacts_for_task(store: PlatformStore, task_id: str, root: Path) -> None: @@ -2862,6 +3086,8 @@ def collect_task_reports(store: PlatformStore, task_id: str) -> dict[str, Any]: "reconstruction_report.json", "validation_report.json", "motrixsim_asset_import_report.json", + "motrixsim_scene_import_report.json", + "runtime_validation.json", "splat_optimization_report.json", "prepared_manifest.json", "dataset_manifest.json", @@ -2876,7 +3102,7 @@ def collect_task_reports(store: PlatformStore, task_id: str) -> dict[str, Any]: if artifact_path.suffix.lower() == ".json" and artifact_path not in candidates: candidates.append(artifact_path) for path in candidates: - if path.name not in report_names: + if path.name not in report_names and not (path.name.startswith("asset_") and path.name.endswith("_validation.json")): continue payload = _read_optional_json(path) failure_summary = _report_failure_summary(path.name, payload) diff --git a/src/real2sim_platform/motrix_scene.py b/src/real2sim_platform/motrix_scene.py new file mode 100644 index 0000000..48d6901 --- /dev/null +++ b/src/real2sim_platform/motrix_scene.py @@ -0,0 +1,691 @@ +from __future__ import annotations + +import hashlib +import json +import math +import os +import shutil +import time +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path, PurePosixPath +from typing import Any, Callable + + +SPATIAL_BUNDLE_SCHEMA = "real2sim.spatial_scene_bundle.v1" +SCENE_MANIFEST_SCHEMA = "real2sim.scene_manifest.v3" +MOTRIX_IMPORT_SCHEMA = "real2sim.motrixsim_scene.v2" +SUPPORTED_MESH_SUFFIXES = {".obj", ".stl", ".ply", ".glb", ".gltf", ".usd", ".usda", ".usdc", ".usdz"} +SUPPORTED_PRIMITIVES = {"box", "sphere", "cylinder", "capsule"} + + +def detect_spatial_bundle(root: Path) -> dict[str, Any]: + root = root.resolve() + files = [root] if root.is_file() else [path for path in root.rglob("*") if path.is_file()] + by_name: dict[str, list[Path]] = {} + for path in files: + by_name.setdefault(path.name.lower(), []).append(path) + + collider = _first_named(by_name, "environment_collider.usdz") + shaded = _first_named(by_name, "scene_shaded.usdz") + splat = _first_named(by_name, "full_res.spz") + if collider is None: + collider = next((path for path in files if path.suffix.lower() == ".usdz" and "collider" in path.stem.lower()), None) + if shaded is None: + shaded = next((path for path in files if path.suffix.lower() == ".usdz" and path != collider), None) + if splat is None: + splat = next((path for path in files if path.suffix.lower() in {".spz", ".splat", ".ksplat", ".sog"}), None) + + missing = [ + label + for label, value in (("environment collider USDZ", collider), ("shaded scene USDZ", shaded), ("3DGS SPZ/SPLAT", splat)) + if value is None + ] + return { + "schema": SPATIAL_BUNDLE_SCHEMA, + "root": str(root), + "ready": not missing, + "missing": missing, + "layers": { + "collision": _file_record(root, collider), + "visual_mesh": _file_record(root, shaded), + "appearance_3dgs": _file_record(root, splat), + }, + "file_count": len(files), + "size_bytes": sum(path.stat().st_size for path in files), + } + + +def build_spatial_scene_package( + input_root: Path, + package_dir: Path, + *, + scene_name: str, + validation_steps: int, + emit: Callable[[str], None] | None = None, +) -> dict[str, Any]: + emit = emit or (lambda _message: None) + detected = detect_spatial_bundle(input_root) + if not detected["ready"]: + raise ValueError("MotrixSim spatial bundle is incomplete: " + ", ".join(detected["missing"])) + + package_dir.mkdir(parents=True, exist_ok=False) + source_dir = package_dir / "source" + mesh_dir = package_dir / "assets" / "meshes" + viewer_dir = package_dir / "viewer" + simulator_dir = package_dir / "simulator_assets" + reports_dir = package_dir / "reports" + for path in (source_dir, mesh_dir, viewer_dir, simulator_dir, reports_dir): + path.mkdir(parents=True, exist_ok=True) + + layers = detected["layers"] + collision_source = Path(layers["collision"]["path"]) + visual_source = Path(layers["visual_mesh"]["path"]) + splat_source = Path(layers["appearance_3dgs"]["path"]) + copied_collision = source_dir / collision_source.name + copied_visual = source_dir / visual_source.name + copied_splat = viewer_dir / splat_source.name + shutil.copy2(collision_source, copied_collision) + shutil.copy2(visual_source, copied_visual) + shutil.copy2(splat_source, copied_splat) + + emit("Reading collision USDZ and converting it to a MotrixSim mesh") + collision_meshes, collision_meta = load_usd_meshes(copied_collision) + if not collision_meshes: + raise ValueError("environment collider USDZ contains no mesh prims") + collision_obj = mesh_dir / "environment_collision.obj" + collision_stats = export_combined_obj(collision_meshes, collision_obj) + + emit("Reading shaded USDZ and building a browser/MotrixSim mesh fallback") + visual_meshes, visual_meta = load_usd_meshes(copied_visual) + if not visual_meshes: + raise ValueError("shaded scene USDZ contains no mesh prims") + visual_obj = mesh_dir / "environment_visual.obj" + visual_stats = export_combined_obj(visual_meshes, visual_obj) + visual_glb = viewer_dir / "environment_visual.glb" + glb_stats = export_glb(visual_meshes, visual_glb) + + runtime_scene = simulator_dir / "scene.xml" + bounds = _combined_bounds(collision_meshes + visual_meshes) + write_runtime_scene(runtime_scene, scene_name=scene_name, collision_obj=collision_obj, visual_obj=visual_obj) + + emit("Loading the generated MJCF with MuJoCo and MotrixSim") + validation = validate_runtime_scene(runtime_scene, steps=validation_steps) + validation_path = reports_dir / "runtime_validation.json" + _write_json(validation_path, validation) + if not validation["ok"]: + failed = [name for name in ("mujoco", "motrixsim") if not validation[name]["ok"]] + raise RuntimeError("Imported scene failed runtime validation: " + "; ".join(f"{name}: {validation[name].get('error', 'unknown error')}" for name in failed)) + + descriptor = { + "schema": MOTRIX_IMPORT_SCHEMA, + "name": scene_name, + "coordinate_frame": {"up_axis": visual_meta["up_axis"], "meters_per_unit": visual_meta["meters_per_unit"]}, + "runtime_scene": str(runtime_scene), + "layers": { + "collision": {"source": str(copied_collision), "runtime_mesh": str(collision_obj), "stats": collision_stats}, + "visual_mesh": {"source": str(copied_visual), "runtime_mesh": str(visual_obj), "browser_mesh": str(visual_glb), "stats": visual_stats}, + "appearance_3dgs": {"source": str(copied_splat), "format": copied_splat.suffix.lower().lstrip(".")}, + }, + "bounds_m": bounds, + "runtime_validation": validation, + "added_assets": [], + "created_at": _utc_now(), + } + descriptor_path = package_dir / "motrix_import.json" + _write_json(descriptor_path, descriptor) + + import_report = { + "schema": "real2sim.motrixsim_scene_import_report.v1", + "ok": True, + "scene_name": scene_name, + "source": detected, + "usd": {"collision": collision_meta, "visual": visual_meta}, + "conversion": {"collision": collision_stats, "visual": visual_stats, "browser_glb": glb_stats}, + "runtime_validation": validation, + "bounds_m": bounds, + "next_actions": ["Add robots or props from the digital asset library", "Save edits as a new scene revision", "Run simulator validation before collection"], + } + report_path = reports_dir / "motrixsim_scene_import_report.json" + _write_json(report_path, import_report) + + manifest = { + "schema": SCENE_MANIFEST_SCHEMA, + "name": scene_name, + "source_type": "motrixsim_spatial_bundle", + "renderer": "spark", + "renderer_status": {"selected": "spark", "spark_available": True, "fallback": "environment_visual.glb"}, + "coordinate_frame": descriptor["coordinate_frame"], + "bounds_m": bounds, + "layers": { + "appearance": "3DGS SPZ/SPLAT", + "visual_mesh": "USDZ shaded mesh", + "collision": "USDZ environment collider", + "simulation": "MotrixSim MJCF", + }, + "assets": [ + _manifest_asset(copied_splat, "scene_appearance", "splat_or_pointcloud", autoload=True), + _manifest_asset(visual_glb, "visual_fallback", "mesh", autoload=False), + _manifest_asset(collision_obj, "environment_collision", "mesh", autoload=False), + _manifest_asset(runtime_scene, "motrixsim_runtime", "simulator", autoload=False), + _manifest_asset(copied_visual, "source_visual_usdz", "source", autoload=False), + _manifest_asset(copied_collision, "source_collision_usdz", "source", autoload=False), + _manifest_asset(validation_path, "runtime_report", "metadata", autoload=False), + ], + "simulator_artifacts": { + "motrixsim": {"path": str(descriptor_path), "runtime_scene": str(runtime_scene), "validated": validation["motrixsim"]["ok"]}, + "mujoco": {"path": str(runtime_scene), "validated": validation["mujoco"]["ok"]}, + }, + "asset_editing": { + "enabled": True, + "revisioned": True, + "supported_primitives": sorted(SUPPORTED_PRIMITIVES), + "supported_mesh_formats": sorted(SUPPORTED_MESH_SUFFIXES), + }, + "metrics": { + "mesh_objects": len(visual_meshes), + "visual_vertices": visual_stats["vertices"], + "visual_triangles": visual_stats["triangles"], + "collision_vertices": collision_stats["vertices"], + "collision_triangles": collision_stats["triangles"], + "runtime_steps": validation_steps, + }, + "created_at": _utc_now(), + } + manifest_path = package_dir / "scene_manifest.json" + _write_json(manifest_path, manifest) + return { + "package_dir": str(package_dir), + "manifest_path": str(manifest_path), + "descriptor_path": str(descriptor_path), + "runtime_scene": str(runtime_scene), + "report_path": str(report_path), + "validation_path": str(validation_path), + "manifest": manifest, + "report": import_report, + } + + +def compose_scene_asset( + package_dir: Path, + *, + asset_spec: dict[str, Any], + source_path: Path | None, + validation_steps: int, + emit: Callable[[str], None] | None = None, +) -> dict[str, Any]: + emit = emit or (lambda _message: None) + manifest_path = package_dir / "scene_manifest.json" + manifest = _read_json(manifest_path) + runtime_scene = _runtime_scene_from_manifest(package_dir, manifest) + if not runtime_scene.is_file(): + raise FileNotFoundError(f"MotrixSim runtime scene not found: {runtime_scene}") + + name = str(asset_spec.get("name") or (source_path.stem if source_path else asset_spec.get("primitive")) or "digital_asset").strip() + asset_key = _safe_name(f"{name}-{hashlib.sha1(json.dumps(asset_spec, sort_keys=True, default=str).encode()).hexdigest()[:8]}") + transform = normalize_transform(asset_spec.get("transform") or {}) + physics = normalize_physics(asset_spec.get("physics") or {}) + primitive = str(asset_spec.get("primitive") or "").lower().strip() + if primitive and primitive not in SUPPORTED_PRIMITIVES: + raise ValueError(f"unsupported primitive: {primitive}") + if not primitive and source_path is None: + raise ValueError("scene asset requires a primitive or a registered source file") + + tree = ET.parse(runtime_scene) + root = tree.getroot() + if root.tag != "mujoco": + raise ValueError("scene runtime file is not an MJCF document") + asset_node = root.find("asset") + if asset_node is None: + asset_node = ET.Element("asset") + root.insert(2, asset_node) + worldbody = root.find("worldbody") + if worldbody is None: + worldbody = ET.SubElement(root, "worldbody") + + body = ET.SubElement( + worldbody, + "body", + { + "name": asset_key, + "pos": _numbers(transform["translation"]), + "euler": _numbers(transform["rotation_euler_degrees"]), + }, + ) + if physics["dynamic"]: + ET.SubElement(body, "freejoint", {"name": f"{asset_key}_joint"}) + + copied_asset: Path | None = None + viewer_asset: Path | None = None + geom_attributes = { + "name": f"{asset_key}_geom", + "friction": _numbers(physics["friction"]), + "contype": "0" if physics["collision_mode"] == "visual_only" else "1", + "conaffinity": "0" if physics["collision_mode"] == "visual_only" else "1", + "group": "2" if physics["collision_mode"] == "visual_only" else "3", + "rgba": _numbers(physics["rgba"]), + } + if physics["dynamic"]: + geom_attributes["mass"] = f"{physics['mass']:.9g}" + + if primitive: + scaled_dimensions = [ + dimension * scale + for dimension, scale in zip(physics["dimensions"], transform["scale"], strict=True) + ] + geom_attributes.update(_primitive_geom(primitive, scaled_dimensions)) + else: + emit(f"Converting {source_path.name} into a MotrixSim mesh") + added_dir = package_dir / "assets" / "added" / asset_key + added_dir.mkdir(parents=True, exist_ok=True) + copied_asset = added_dir / source_path.name + shutil.copy2(source_path, copied_asset) + runtime_mesh = added_dir / f"{asset_key}.obj" + convert_asset_to_obj(copied_asset, runtime_mesh) + viewer_asset = copied_asset if copied_asset.suffix.lower() in {".glb", ".gltf", ".obj"} else runtime_mesh + compiler = root.find("compiler") + meshdir = Path((compiler.attrib.get("meshdir") if compiler is not None else "") or ".") + resolved_meshdir = (runtime_scene.parent / meshdir).resolve() + relative_mesh = os.path.relpath(runtime_mesh, resolved_meshdir).replace(os.sep, "/") + mesh_name = f"{asset_key}_mesh" + ET.SubElement(asset_node, "mesh", {"name": mesh_name, "file": relative_mesh, "scale": _numbers(transform["scale"])} ) + geom_attributes.update({"type": "mesh", "mesh": mesh_name}) + ET.SubElement(body, "geom", geom_attributes) + _indent_xml(root) + tree.write(runtime_scene, encoding="utf-8", xml_declaration=True) + + emit("Validating the revised scene with MuJoCo and MotrixSim") + validation = validate_runtime_scene(runtime_scene, steps=validation_steps) + validation_path = package_dir / "reports" / f"asset_{asset_key}_validation.json" + validation_path.parent.mkdir(parents=True, exist_ok=True) + _write_json(validation_path, validation) + if not validation["ok"]: + failed = [name for name in ("mujoco", "motrixsim") if not validation[name]["ok"]] + raise RuntimeError("Added asset failed runtime validation: " + "; ".join(f"{name}: {validation[name].get('error', 'unknown error')}" for name in failed)) + + entry = { + "name": name, + "path": str(viewer_asset or ""), + "kind": "primitive" if primitive else "mesh", + "role": "added_asset", + "format": primitive or (viewer_asset.suffix.lower().lstrip(".") if viewer_asset else ""), + "autoload": True, + "viewer_hint": "overlay", + "viewer_primitive": primitive or None, + "transform": transform, + "physics": physics, + "runtime": {"body": asset_key, "scene": str(runtime_scene), "validated": validation["motrixsim"]["ok"]}, + "added_at": _utc_now(), + } + if viewer_asset: + entry["size_bytes"] = viewer_asset.stat().st_size + + assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and item.get("runtime", {}).get("body") == asset_key)] + manifest["assets"] = [entry] + assets + manifest["updated_at"] = _utc_now() + manifest["asset_editing"] = { + **(manifest.get("asset_editing") or {}), + "enabled": True, + "revisioned": True, + "last_added_asset": name, + } + manifest.setdefault("simulator_artifacts", {}).setdefault("motrixsim", {})["validated"] = validation["motrixsim"]["ok"] + manifest.setdefault("simulator_artifacts", {}).setdefault("mujoco", {})["validated"] = validation["mujoco"]["ok"] + _write_json(manifest_path, manifest) + + descriptor_path = package_dir / "motrix_import.json" + descriptor = _read_json(descriptor_path) if descriptor_path.is_file() else {"schema": MOTRIX_IMPORT_SCHEMA, "runtime_scene": str(runtime_scene), "added_assets": []} + descriptor.setdefault("added_assets", []).append(entry) + descriptor["runtime_validation"] = validation + descriptor["updated_at"] = _utc_now() + _write_json(descriptor_path, descriptor) + return { + "manifest_path": str(manifest_path), + "runtime_scene": str(runtime_scene), + "validation_path": str(validation_path), + "validation": validation, + "asset": entry, + } + + +def load_usd_meshes(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: + try: + import numpy as np + from pxr import Gf, Usd, UsdGeom + except ImportError as exc: + raise RuntimeError("USD conversion requires usd-core and numpy") from exc + + stage = Usd.Stage.Open(str(path)) + if stage is None: + raise ValueError(f"unable to open USD stage: {path}") + up_axis = str(stage.GetMetadata("upAxis") or "Z") + meters_per_unit = float(stage.GetMetadata("metersPerUnit") or 1.0) + cache = UsdGeom.XformCache() + meshes: list[dict[str, Any]] = [] + for prim in stage.Traverse(): + if not prim.IsA(UsdGeom.Mesh): + continue + mesh = UsdGeom.Mesh(prim) + raw_points = mesh.GetPointsAttr().Get() or [] + counts = np.asarray(mesh.GetFaceVertexCountsAttr().Get() or [], dtype=np.int64) + indices = np.asarray(mesh.GetFaceVertexIndicesAttr().Get() or [], dtype=np.int64) + if not raw_points or not len(counts) or not len(indices): + continue + points = np.asarray(raw_points, dtype=np.float64) + matrix = cache.GetLocalToWorldTransform(prim) + transformed = np.asarray([matrix.Transform(Gf.Vec3d(*point)) for point in points], dtype=np.float64) + transformed *= meters_per_unit + faces = triangulate_faces(counts, indices) + meshes.append({"name": _safe_name(prim.GetName()), "vertices": transformed, "faces": faces}) + return meshes, { + "path": str(path), + "up_axis": up_axis, + "meters_per_unit": meters_per_unit, + "mesh_count": len(meshes), + "vertices": sum(len(item["vertices"]) for item in meshes), + "triangles": sum(len(item["faces"]) for item in meshes), + } + + +def triangulate_faces(counts: Any, indices: Any) -> Any: + import numpy as np + + if len(counts) and bool(np.all(counts == 3)): + return np.asarray(indices, dtype=np.int64).reshape((-1, 3)) + triangles: list[tuple[int, int, int]] = [] + cursor = 0 + for count in counts: + polygon = indices[cursor : cursor + int(count)] + cursor += int(count) + if len(polygon) < 3: + continue + triangles.extend((int(polygon[0]), int(polygon[index]), int(polygon[index + 1])) for index in range(1, len(polygon) - 1)) + return np.asarray(triangles, dtype=np.int64) + + +def export_combined_obj(meshes: list[dict[str, Any]], output: Path) -> dict[str, Any]: + import numpy as np + + output.parent.mkdir(parents=True, exist_ok=True) + vertices: list[Any] = [] + faces: list[Any] = [] + offset = 0 + for mesh in meshes: + mesh_vertices = np.asarray(mesh["vertices"], dtype=np.float64) + mesh_faces = np.asarray(mesh["faces"], dtype=np.int64) + vertices.append(mesh_vertices) + faces.append(mesh_faces + offset) + offset += len(mesh_vertices) + all_vertices = np.concatenate(vertices, axis=0) + all_faces = np.concatenate(faces, axis=0) + with output.open("w", encoding="ascii", newline="\n") as handle: + handle.write("# Real2Sim USD mesh conversion\n") + np.savetxt(handle, all_vertices, fmt="v %.9g %.9g %.9g") + np.savetxt(handle, all_faces + 1, fmt="f %d %d %d") + return { + "path": str(output), + "vertices": int(len(all_vertices)), + "triangles": int(len(all_faces)), + "size_bytes": output.stat().st_size, + "sha256": _sha256(output), + } + + +def export_glb(meshes: list[dict[str, Any]], output: Path) -> dict[str, Any]: + try: + import numpy as np + import trimesh + except ImportError as exc: + raise RuntimeError("GLB conversion requires trimesh and numpy") from exc + + scene = trimesh.Scene() + palette = ([55, 170, 155, 255], [78, 132, 175, 255], [172, 132, 82, 255], [138, 112, 168, 255]) + for index, item in enumerate(meshes): + mesh = trimesh.Trimesh(vertices=item["vertices"], faces=item["faces"], process=False, validate=False) + mesh.visual.vertex_colors = np.tile(np.asarray(palette[index % len(palette)], dtype=np.uint8), (len(mesh.vertices), 1)) + scene.add_geometry(mesh, geom_name=f"{item['name']}_{index}", node_name=f"{item['name']}_{index}") + output.parent.mkdir(parents=True, exist_ok=True) + scene.export(output, file_type="glb") + return {"path": str(output), "size_bytes": output.stat().st_size, "sha256": _sha256(output), "mesh_count": len(meshes)} + + +def convert_asset_to_obj(source: Path, output: Path) -> None: + suffix = source.suffix.lower() + if suffix == ".obj": + shutil.copy2(source, output) + return + if suffix in {".usd", ".usda", ".usdc", ".usdz"}: + meshes, _metadata = load_usd_meshes(source) + if not meshes: + raise ValueError(f"USD asset contains no meshes: {source.name}") + export_combined_obj(meshes, output) + return + if suffix not in SUPPORTED_MESH_SUFFIXES: + raise ValueError(f"unsupported MotrixSim mesh asset format: {suffix or 'unknown'}") + try: + import trimesh + except ImportError as exc: + raise RuntimeError("mesh conversion requires trimesh") from exc + loaded = trimesh.load(source, force="scene", process=False) + if isinstance(loaded, trimesh.Trimesh): + mesh = loaded + else: + mesh = loaded.to_geometry() + if mesh is None or not isinstance(mesh, trimesh.Trimesh) or mesh.is_empty: + raise ValueError(f"mesh asset contains no geometry: {source.name}") + output.parent.mkdir(parents=True, exist_ok=True) + mesh.export(output, file_type="obj") + + +def write_runtime_scene(path: Path, *, scene_name: str, collision_obj: Path, visual_obj: Path) -> None: + root = ET.Element("mujoco", {"model": _safe_name(scene_name)}) + ET.SubElement(root, "compiler", {"angle": "degree", "meshdir": "../assets/meshes", "autolimits": "true"}) + ET.SubElement(root, "option", {"timestep": "0.002", "gravity": "0 0 -9.81", "integrator": "implicitfast"}) + asset = ET.SubElement(root, "asset") + ET.SubElement(asset, "mesh", {"name": "environment_collision", "file": collision_obj.name}) + ET.SubElement(asset, "mesh", {"name": "environment_visual", "file": visual_obj.name}) + worldbody = ET.SubElement(root, "worldbody") + ET.SubElement(worldbody, "light", {"name": "key_light", "pos": "2 -2 6", "dir": "-0.2 0.2 -1", "diffuse": "0.9 0.9 0.9"}) + environment = ET.SubElement(worldbody, "body", {"name": "imported_environment"}) + ET.SubElement(environment, "geom", {"name": "environment_visual", "type": "mesh", "mesh": "environment_visual", "contype": "0", "conaffinity": "0", "group": "2", "rgba": "0.72 0.76 0.78 1"}) + ET.SubElement(environment, "geom", {"name": "environment_collision", "type": "mesh", "mesh": "environment_collision", "contype": "1", "conaffinity": "1", "group": "3", "friction": "0.9 0.01 0.001", "rgba": "0.2 0.7 0.65 0"}) + _indent_xml(root) + path.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(root).write(path, encoding="utf-8", xml_declaration=True) + + +def validate_runtime_scene(path: Path, *, steps: int) -> dict[str, Any]: + result = {"steps": steps, "mujoco": {"ok": False}, "motrixsim": {"ok": False}} + try: + import mujoco + + model = mujoco.MjModel.from_xml_path(str(path)) + data = mujoco.MjData(model) + for _ in range(steps): + mujoco.mj_step(model, data) + result["mujoco"] = {"ok": True, "status": "loaded_and_stepped", "nbody": int(model.nbody), "ngeom": int(model.ngeom), "steps": steps} + except Exception as exc: + result["mujoco"] = {"ok": False, "status": "failed", "error": f"{type(exc).__name__}: {exc}"} + try: + from motrixsim import SceneData, load_model, step + + model = load_model(str(path)) + data = SceneData(model) + for _ in range(steps): + step(model, data) + result["motrixsim"] = { + "ok": True, + "status": "loaded_and_stepped", + "steps": steps, + "timestep": float(model.options.timestep), + "nq": len(data.low.dof_positions), + "nv": len(data.low.dof_velocities), + } + except Exception as exc: + result["motrixsim"] = {"ok": False, "status": "failed", "error": f"{type(exc).__name__}: {exc}"} + result["ok"] = bool(result["motrixsim"]["ok"] and result["mujoco"]["ok"]) + return result + + +def normalize_transform(value: dict[str, Any]) -> dict[str, list[float]]: + translation = _vector(value.get("translation"), 3, 0.0) + rotation = _vector(value.get("rotation_euler_degrees"), 3, 0.0) + if value.get("yaw_degrees") is not None and not any(rotation): + rotation[2] = float(value["yaw_degrees"]) + scale_value = value.get("scale", 1.0) + scale = [float(scale_value)] * 3 if isinstance(scale_value, (int, float)) else _vector(scale_value, 3, 1.0) + if any(component <= 0 for component in scale): + raise ValueError("asset scale must be positive") + return {"translation": translation, "rotation_euler_degrees": rotation, "scale": scale} + + +def normalize_physics(value: dict[str, Any]) -> dict[str, Any]: + collision_mode = str(value.get("collision_mode") or "visual_and_collision") + if collision_mode not in {"visual_only", "collision_only", "visual_and_collision"}: + raise ValueError("collision_mode must be visual_only, collision_only, or visual_and_collision") + mass = float(value.get("mass", 0.25)) + if mass <= 0: + raise ValueError("asset mass must be positive") + dimensions = _vector(value.get("dimensions"), 3, 0.1) + if any(component <= 0 for component in dimensions): + raise ValueError("asset dimensions must be positive") + rgba = _vector(value.get("rgba"), 4, 0.0) + if not value.get("rgba"): + rgba = [0.13, 0.76, 0.67, 1.0 if collision_mode != "collision_only" else 0.2] + return { + "dynamic": bool(value.get("dynamic", False)), + "mass": mass, + "collision_mode": collision_mode, + "dimensions": dimensions, + "friction": _vector(value.get("friction"), 3, 0.0) if value.get("friction") else [0.8, 0.01, 0.001], + "rgba": rgba, + } + + +def _primitive_geom(primitive: str, dimensions: list[float]) -> dict[str, str]: + x, y, z = dimensions + if primitive == "box": + size = [x / 2, y / 2, z / 2] + elif primitive == "sphere": + size = [max(x, y, z) / 2] + else: + size = [max(x, y) / 2, z / 2] + return {"type": primitive, "size": _numbers(size)} + + +def _runtime_scene_from_manifest(package_dir: Path, manifest: dict[str, Any]) -> Path: + motrix = (manifest.get("simulator_artifacts") or {}).get("motrixsim") or {} + candidate = Path(str(motrix.get("runtime_scene") or "")) + if candidate.is_file(): + return candidate + for relative in ("simulator_assets/scene.xml", "scene.xml", "scene_mujoco.xml"): + path = package_dir / relative + if path.is_file(): + return path + return package_dir / "simulator_assets" / "scene.xml" + + +def _manifest_asset(path: Path, role: str, kind: str, *, autoload: bool) -> dict[str, Any]: + return { + "name": path.name, + "path": str(path), + "role": role, + "kind": kind, + "format": path.suffix.lower().lstrip("."), + "size_bytes": path.stat().st_size, + "autoload": autoload, + "viewer_hint": "autoload" if autoload else "manual_load", + "sha256": _sha256(path), + } + + +def _combined_bounds(meshes: list[dict[str, Any]]) -> dict[str, list[float]]: + import numpy as np + + minimum = np.min(np.vstack([mesh["vertices"].min(axis=0) for mesh in meshes]), axis=0) + maximum = np.max(np.vstack([mesh["vertices"].max(axis=0) for mesh in meshes]), axis=0) + return {"min": [float(value) for value in minimum], "max": [float(value) for value in maximum]} + + +def _first_named(by_name: dict[str, list[Path]], name: str) -> Path | None: + values = by_name.get(name.lower()) or [] + return values[0] if values else None + + +def _file_record(root: Path, path: Path | None) -> dict[str, Any] | None: + if path is None: + return None + return { + "path": str(path), + "relative_path": path.relative_to(root).as_posix() if root.is_dir() else path.name, + "size_bytes": path.stat().st_size, + "sha256": _sha256(path), + "format": path.suffix.lower().lstrip("."), + } + + +def _safe_name(value: str) -> str: + normalized = "".join(character if character.isalnum() or character in {"_", "-"} else "_" for character in value.strip()) + return normalized.strip("_-")[:96] or "asset" + + +def _vector(value: Any, length: int, default: float) -> list[float]: + if value is None: + return [default] * length + if not isinstance(value, (list, tuple)) or len(value) != length: + raise ValueError(f"expected a {length}-component vector") + result = [float(component) for component in value] + if any(not math.isfinite(component) for component in result): + raise ValueError("vector components must be finite") + return result + + +def _numbers(values: list[float]) -> str: + return " ".join(f"{value:.9g}" for value in values) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON document must be an object: {path}") + return payload + + +def _utc_now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def _indent_xml(root: ET.Element) -> None: + try: + ET.indent(root, space=" ") + except AttributeError: + pass + + +def safe_extract_zip(archive: zipfile.ZipFile, target: Path, *, max_total_bytes: int = 4 * 1024 * 1024 * 1024) -> None: + total = 0 + target_resolved = target.resolve() + for info in archive.infolist(): + pure = PurePosixPath(info.filename.replace("\\", "/")) + if pure.is_absolute() or ".." in pure.parts: + raise ValueError(f"unsafe archive path: {info.filename}") + total += int(info.file_size) + if total > max_total_bytes: + raise ValueError("archive expands beyond the configured safety limit") + destination = (target / Path(*pure.parts)).resolve() + if destination != target_resolved and target_resolved not in destination.parents: + raise ValueError(f"archive path escapes target directory: {info.filename}") + archive.extractall(target) diff --git a/src/real2sim_platform/server.py b/src/real2sim_platform/server.py index d92e139..21edba6 100644 --- a/src/real2sim_platform/server.py +++ b/src/real2sim_platform/server.py @@ -21,12 +21,13 @@ from urllib.parse import urlparse from fastapi import FastAPI, File, HTTPException, Request, Response, UploadFile from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, StreamingResponse from fastapi.staticfiles import StaticFiles -from pydantic import BaseModel +from pydantic import BaseModel, Field from . import __version__ from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config +from .motrix_scene import SUPPORTED_MESH_SUFFIXES, SUPPORTED_PRIMITIVES from .source_revision import read_source_revision from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file from .store import PlatformStore @@ -45,11 +46,16 @@ class ImportSceneRequest(BaseModel): class SceneAssetRequest(BaseModel): - path: str + path: str = "" + source: dict[str, Any] = Field(default_factory=dict) + primitive: str | None = None name: str | None = None role: str = "added_asset" kind: str | None = None - transform: dict[str, Any] = {} + transform: dict[str, Any] = Field(default_factory=dict) + physics: dict[str, Any] = Field(default_factory=dict) + validation_steps: int = 60 + revision_name: str | None = None class CreateTaskRequest(BaseModel): @@ -299,6 +305,8 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: "aholo_optional": True, "mujoco": True, "motrixsim": True, + "motrixsim_scene_import": True, + "scene_asset_compose": True, }, } @@ -530,6 +538,21 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: def list_datasets() -> list[dict[str, Any]]: return [_with_view_model(item, "dataset") for item in store.list_datasets()] + @app.post("/api/datasets/{dataset_id}/import-motrixsim") + def import_dataset_to_motrixsim(dataset_id: str) -> dict[str, Any]: + try: + dataset = store.get_dataset(dataset_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="dataset not found") from exc + if not bool((dataset.get("scan") or {}).get("spatial_bundle_ready")): + raise HTTPException(status_code=400, detail="该数据集不包含完整的 USDZ 碰撞层、USDZ 视觉层和 SPZ/SPLAT 外观层") + return _with_task_view_model( + _queue_task( + "motrixsim_scene_import", + {"dataset_id": dataset_id, "scene_name": f"{dataset['name']} · MotrixSim 场景", "validation_steps": 120}, + ) + ) + @app.post("/api/datasets/import-path") def import_dataset(payload: ImportPathRequest) -> dict[str, Any]: path = Path(payload.path) @@ -567,6 +590,10 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: def list_scenes() -> list[dict[str, Any]]: return [_with_view_model(item, "scene") for item in store.list_scenes()] + @app.get("/api/assets") + def list_digital_assets() -> list[dict[str, Any]]: + return _digital_asset_catalog(store) + @app.post("/api/scenes/import") def import_scene(payload: ImportSceneRequest) -> dict[str, Any]: path = Path(payload.path) @@ -610,57 +637,18 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: @app.post("/api/scenes/{scene_id}/assets") def add_scene_asset(scene_id: str, payload: SceneAssetRequest) -> dict[str, Any]: try: - scene = store.get_scene(scene_id) + store.get_scene(scene_id) except KeyError as exc: raise HTTPException(status_code=404, detail="scene not found") from exc - asset_path = Path(payload.path).expanduser() - if not asset_path.is_file(): - raise HTTPException(status_code=404, detail="asset file not found") - if not is_path_allowed(asset_path, cfg): - raise HTTPException(status_code=403, detail="asset path is outside configured import whitelist") - revision_id = secrets.token_hex(16) - revision_dir = cfg.scenes_dir / "revisions" / revision_id - source_scene_path = Path(scene["path"]) - source_package = source_scene_path if source_scene_path.is_dir() else source_scene_path.parent - _copy_scene_revision(source_package, revision_dir) - revision_scene_path = revision_dir / source_scene_path.name if source_scene_path.is_file() else revision_dir - source_manifest_path = _scene_manifest_path_for_write(scene) - manifest_path = revision_dir / source_manifest_path.name - manifest = _read_manifest_or_default(scene, source_manifest_path) - artifact = store.add_artifact(task_id=None, name=payload.name or asset_path.name, path=asset_path, kind=(payload.kind or _scene_asset_kind(asset_path))) - asset = _scene_asset_entry( - asset_path, - url=f"/api/artifacts/{artifact['id']}/content", - name=payload.name, - kind=payload.kind, - role=payload.role, - transform=payload.transform, - ) - assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and str(item.get("path") or "") == str(asset_path))] - manifest["assets"] = [asset] + assets - manifest["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - manifest["revision"] = {"id": revision_id, "parent_scene_id": scene_id, "reason": "asset_added"} - manifest["asset_editing"] = {"supports_added_assets": True, "last_added_asset": asset["name"]} - manifest_path.parent.mkdir(parents=True, exist_ok=True) - manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset", "scene_revision"])) - metadata = dict(scene.get("metadata") or {}) - metadata.update({"parent_scene_id": scene_id, "revision_reason": "asset_added", "package": str(revision_dir)}) - revision = store.create_scene( - item_id=revision_id, - name=f"{scene['name']} · 资产修订", - path=revision_scene_path, - metadata=metadata, - source_type="scene_revision", - source_uri=scene.get("storage_uri") or scene.get("source_uri") or str(source_scene_path), - version=_next_revision_label(str(scene.get("version") or "")), - parent_task_id=scene.get("parent_task_id"), - manifest_path=manifest_path, - metrics=scene.get("metrics") or {}, - tags=tags, - cache_state="local", - ) - return {"ok": True, "scene": revision, "parent_scene_id": scene_id, "asset": asset, "manifest_path": str(manifest_path)} + params = payload.model_dump(exclude_none=True) + params["scene_id"] = scene_id + task = _queue_task("scene_asset_compose", params) + return { + "ok": True, + "scene_id": scene_id, + "task": _with_task_view_model(task), + "message": "数字资产合成任务已创建;通过 MotrixSim 与 MuJoCo 校验后会生成新场景版本", + } @app.patch("/api/scenes/{scene_id}") def update_scene(scene_id: str, payload: UpdateEntityRequest) -> dict[str, Any]: @@ -708,7 +696,7 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: payload = CreateTaskRequest(type=task_type, params=params) _validate_task_payload(payload, cfg) _validate_task_resources(store, payload) - if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "motrixsim_asset_import", "simulator_validation"}: + if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation"}: raise ValueError(f"unknown task type: {payload.type}") run_dir = cfg.runs_dir / payload.type / _next_run_name(cfg.runs_dir / payload.type, payload.type) return store.create_task(task_type=payload.type, params=payload.params, run_dir=run_dir) @@ -1181,7 +1169,7 @@ def _metrics_summary(kind: str, metrics: dict[str, Any], scan: dict[str, Any], m def _resource_subtitle(kind: str, entity: dict[str, Any], manifest: dict[str, Any]) -> str: source = str(entity.get("source_type") or manifest.get("source_type") or "") version = str(entity.get("version") or "") - label = {"prepared_media": "已准备帧数据", "expert_collection": "专家采集数据", "scene_reconstruction": "场景重建", "huggingface": "HuggingFace", "upload": "上传文件", "local_path": "本地导入"}.get(source, source or kind) + label = {"prepared_media": "已准备帧数据", "expert_collection": "专家采集数据", "scene_reconstruction": "场景重建", "motrixsim_spatial_bundle": "MotrixSim 空间包", "scene_revision": "场景修订", "huggingface": "HuggingFace", "upload": "上传文件", "browser_upload": "浏览器上传", "local_path": "本地导入"}.get(source, source or kind) return f"{label}{' · ' + version if version else ''}" @@ -1197,6 +1185,8 @@ def _task_type_label(task_type: str) -> str: "reconstruction_preflight": "重建预检", "scene_reconstruction": "场景重建", "motrixsim_asset_import": "MotrixSim 资产导入", + "motrixsim_scene_import": "MotrixSim 场景包导入", + "scene_asset_compose": "数字资产合成", "simulator_validation": "仿真校验", "expert_collection": "专家采集", "training": "模型训练", @@ -1220,6 +1210,30 @@ def _validate_task_payload(payload: CreateTaskRequest, cfg: PlatformConfig) -> N raise HTTPException(status_code=400, detail="validation_steps must be between 1 and 5000") if not 1 <= int(params.get("max_parlour_meshes", 240)) <= 1000: raise HTTPException(status_code=400, detail="max_parlour_meshes must be between 1 and 1000") + if payload.type == "motrixsim_scene_import": + if not params.get("dataset_id"): + raise HTTPException(status_code=400, detail="motrixsim_scene_import requires dataset_id") + if not 1 <= int(params.get("validation_steps", 120)) <= 5000: + raise HTTPException(status_code=400, detail="validation_steps must be between 1 and 5000") + if payload.type == "scene_asset_compose": + if not params.get("scene_id"): + raise HTTPException(status_code=400, detail="scene_asset_compose requires scene_id") + source = params.get("source") or {} + primitive = str(params.get("primitive") or "").lower() + if primitive and primitive not in SUPPORTED_PRIMITIVES: + raise HTTPException(status_code=400, detail=f"unsupported primitive: {primitive}") + if not primitive and not (source.get("type") in {"dataset", "scene"} or params.get("path")): + raise HTTPException(status_code=400, detail="scene_asset_compose requires a primitive or registered asset source") + if source.get("type") == "dataset" and not (source.get("dataset_id") and source.get("relative_path")): + raise HTTPException(status_code=400, detail="dataset asset source requires dataset_id and relative_path") + if not 1 <= int(params.get("validation_steps", 60)) <= 5000: + raise HTTPException(status_code=400, detail="validation_steps must be between 1 and 5000") + if params.get("path"): + path = Path(str(params["path"])).expanduser() + if not path.is_file(): + raise HTTPException(status_code=400, detail="asset file not found") + if not is_path_allowed(path, cfg): + raise HTTPException(status_code=403, detail="asset path is outside configured import roots") if payload.type == "hf_import": repo_type = params.get("repo_type", "dataset") if repo_type != "dataset": @@ -1292,6 +1306,21 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) - try: if payload.type == "expert_collection" and params.get("scene_id"): store.get_scene(str(params["scene_id"])) + if payload.type == "motrixsim_scene_import" and params.get("dataset_id"): + dataset = store.get_dataset(str(params["dataset_id"])) + if not bool((dataset.get("scan") or {}).get("spatial_bundle_ready")): + raise HTTPException(status_code=400, detail="数据集不是完整的 MotrixSim USDZ + SPZ 空间包") + if payload.type == "scene_asset_compose": + store.get_scene(str(params["scene_id"])) + source = params.get("source") or {} + if source.get("type") == "dataset": + dataset = store.get_dataset(str(source.get("dataset_id") or "")) + root = Path(dataset["path"]).resolve() + candidate = (root / str(source.get("relative_path") or "")).resolve() + if candidate != root and root not in candidate.parents: + raise HTTPException(status_code=400, detail="资产路径越出所选数据集") + if not candidate.is_file() or candidate.suffix.lower() not in SUPPORTED_MESH_SUFFIXES: + raise HTTPException(status_code=400, detail="所选数据集文件不是可导入 MotrixSim 的网格资产") if payload.type == "training" and params.get("dataset_id"): dataset = store.get_dataset(str(params["dataset_id"])) source = str(dataset.get("source_type") or "") @@ -1310,6 +1339,58 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) - raise HTTPException(status_code=400, detail=f"任务引用的资源不存在: {exc.args[0]}") from exc +def _digital_asset_catalog(store: PlatformStore) -> list[dict[str, Any]]: + primitive_names = {"box": "立方体", "sphere": "球体", "cylinder": "圆柱体", "capsule": "胶囊体"} + items: list[dict[str, Any]] = [ + { + "id": f"primitive:{primitive}", + "name": primitive_names.get(primitive, primitive), + "source_type": "primitive", + "format": primitive, + "source": {"type": "primitive"}, + "capabilities": {"motrixsim_physics": True, "browser_preview": True, "dynamic": True, "collision": True}, + } + for primitive in sorted(SUPPORTED_PRIMITIVES) + ] + for dataset in store.list_datasets(): + root = Path(dataset["path"]) + if not root.is_dir(): + continue + relative_files = list((dataset.get("scan") or {}).get("scene_asset_files") or []) + if not relative_files: + relative_files = [path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file() and path.suffix.lower() in SUPPORTED_MESH_SUFFIXES][:200] + for relative_text in relative_files[:200]: + relative = Path(str(relative_text)) + candidate = (root / relative).resolve() + root_resolved = root.resolve() + if candidate == root_resolved or root_resolved not in candidate.parents or not candidate.is_file(): + continue + suffix = candidate.suffix.lower() + if suffix not in SUPPORTED_MESH_SUFFIXES: + continue + identifier = hashlib.sha256(f"dataset:{dataset['id']}:{relative.as_posix()}".encode("utf-8")).hexdigest()[:24] + items.append( + { + "id": identifier, + "name": candidate.stem.replace("_", " "), + "source_type": "dataset", + "dataset_id": dataset["id"], + "dataset_name": dataset["name"], + "relative_path": relative.as_posix(), + "format": suffix.lstrip("."), + "size_bytes": candidate.stat().st_size, + "source": {"type": "dataset", "dataset_id": dataset["id"], "relative_path": relative.as_posix()}, + "capabilities": { + "motrixsim_physics": True, + "browser_preview": suffix in {".obj", ".glb", ".gltf", ".ply"}, + "dynamic": True, + "collision": True, + }, + } + ) + return items + + def _active_admin_count(store: PlatformStore) -> int: return sum(1 for user in store.list_users() if user["role"] == "admin" and not user["disabled"]) diff --git a/src/real2sim_platform/static/app.js b/src/real2sim_platform/static/app.js index 0ac32ad..eff5af1 100644 --- a/src/real2sim_platform/static/app.js +++ b/src/real2sim_platform/static/app.js @@ -1,4 +1,4 @@ -const STATIC_VERSION = "20260711-poc-product-v3"; +const STATIC_VERSION = "20260712-motrix-spatial-v1"; const state = { datasets: [], @@ -10,6 +10,7 @@ const state = { currentImport: "hf", editing: null, assetLibrary: [], + digitalAssets: [], refreshTimer: null, taskEvents: null, selections: {}, @@ -24,7 +25,7 @@ const $ = (id) => document.getElementById(id); const taskScopes = { dataTaskList: ["hf_import", "media_prepare"], - sceneTaskList: ["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"], + sceneTaskList: ["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"], collectTaskList: ["expert_collection"], trainTaskList: ["training"], inferTaskList: ["inference"], @@ -124,16 +125,18 @@ function fillRecipeSelect(id, items, emptyLabel) { async function refresh({ quiet = false } = {}) { try { - const [datasets, scenes, models, tasks] = await Promise.all([ + const [datasets, scenes, models, tasks, digitalAssets] = await Promise.all([ api("/api/datasets"), api("/api/scenes"), api("/api/models"), api("/api/tasks"), + api("/api/assets"), ]); state.datasets = Array.isArray(datasets) ? datasets : []; state.scenes = Array.isArray(scenes) ? scenes : []; state.models = Array.isArray(models) ? models : []; state.tasks = Array.isArray(tasks) ? tasks : []; + state.digitalAssets = Array.isArray(digitalAssets) ? digitalAssets : []; if (state.activeSceneId && !state.scenes.some((scene) => scene.id === state.activeSceneId)) { state.activeSceneId = ""; } @@ -279,6 +282,7 @@ function renderDatasetList() { detail: datasetStats(item), actions: [ ["查看", `data-view-json="/api/datasets/${item.id}/manifest"`], + ...(item.scan?.spatial_bundle_ready ? [["导入 MotrixSim", `data-dataset-motrixsim="${item.id}"`]] : []), ["准备", `data-dataset-prepare="${item.id}"`], ["预检", `data-dataset-preflight="${item.id}"`], ], @@ -458,12 +462,13 @@ function fillAssetPreset() { const select = $("assetPreset"); if (!select) return; const previous = select.value; - select.innerHTML = [``] - .concat(state.assetLibrary.slice(0, 80).map((asset) => ``)) + select.innerHTML = [``] + .concat(state.digitalAssets.slice(0, 240).map((asset) => ``)) .join(""); - select.disabled = state.assetLibrary.length === 0; - if (previous && state.assetLibrary.some((asset) => asset.path === previous)) select.value = previous; - else if (state.assetLibrary.length) select.value = state.assetLibrary[0].path; + select.disabled = state.digitalAssets.length === 0; + if (previous && state.digitalAssets.some((asset) => asset.id === previous)) select.value = previous; + else if (state.digitalAssets.length) select.value = state.digitalAssets[0].id; + updateDigitalAssetHint(); } function resourceRow(kind, item, options) { @@ -1031,20 +1036,42 @@ function reconstructionOptions() { } function sceneAssetPayload() { - const preset = $("assetPreset")?.value || ""; + const selected = state.digitalAssets.find((asset) => asset.id === $("assetPreset")?.value); + const primitive = selected?.source_type === "primitive" ? selected.format : undefined; return { - path: $("assetPath")?.value.trim() || preset, - name: $("assetName")?.value.trim() || undefined, - kind: $("assetKind")?.value === "auto" ? undefined : $("assetKind")?.value, - role: $("assetRole")?.value || "added_asset", + source: selected?.source || {}, + primitive, + name: $("assetName")?.value.trim() || selected?.name || undefined, transform: { translation: [Number($("assetPosX")?.value || 0), Number($("assetPosY")?.value || 0), Number($("assetPosZ")?.value || 0)], - scale: Number($("assetScale")?.value || 1), - yaw_degrees: Number($("assetYaw")?.value || 0), + rotation_euler_degrees: [Number($("assetRotX")?.value || 0), Number($("assetRotY")?.value || 0), Number($("assetRotZ")?.value || 0)], + scale: [Number($("assetScaleX")?.value || 1), Number($("assetScaleY")?.value || 1), Number($("assetScaleZ")?.value || 1)], }, + physics: { + dynamic: $("assetDynamic")?.checked || false, + mass: Number($("assetMass")?.value || 0.25), + collision_mode: $("assetCollisionMode")?.value || "visual_and_collision", + dimensions: [Number($("assetSizeX")?.value || 0.1), Number($("assetSizeY")?.value || 0.1), Number($("assetSizeZ")?.value || 0.1)], + }, + validation_steps: Number($("assetValidationSteps")?.value || 60), }; } +function digitalAssetOptionLabel(asset) { + const source = asset.source_type === "primitive" ? "基础形状" : asset.dataset_name || "已上传资产"; + const size = asset.size_bytes ? formatBytes(asset.size_bytes) : ""; + return [asset.name, source, String(asset.format || "").toUpperCase(), size].filter(Boolean).join(" · "); +} + +function updateDigitalAssetHint() { + const asset = state.digitalAssets.find((item) => item.id === $("assetPreset")?.value); + const hint = $("assetSourceHint"); + if (!hint) return; + if (!asset) hint.textContent = "上传 OBJ、STL、GLB、PLY 或 USDZ 后即可选择"; + else if (asset.source_type === "primitive") hint.textContent = `${asset.name}会写入 MotrixSim MJCF,可设置尺寸、质量和碰撞模式`; + else hint.textContent = `${asset.dataset_name || "上传数据"} · ${String(asset.format || "").toUpperCase()} · 将转换为 MotrixSim 网格并真实校验`; +} + function preparedDatasets() { return state.datasets.filter((dataset) => dataset.source_type === "prepared_media"); } @@ -1064,7 +1091,7 @@ function collectionDatasets() { } function sceneIdFromTask(task) { - return task?.type === "scene_reconstruction" ? task.result?.scene_id || "" : ""; + return ["scene_reconstruction", "motrixsim_scene_import", "scene_asset_compose"].includes(task?.type) ? task.result?.scene_id || "" : ""; } function resourceView(kind, item) { @@ -1130,11 +1157,14 @@ function datasetStats(item) { if (item.metrics?.samples) pieces.push(`${item.metrics.samples} 样本`); if (item.scan?.image_count) pieces.push(`${item.scan.image_count} 张图片`); if (item.scan?.video_count) pieces.push(`${item.scan.video_count} 段视频`); + if (item.scan?.usdz_count) pieces.push(`${item.scan.usdz_count} 个 USDZ`); + if (item.scan?.splat_count) pieces.push(`${item.scan.splat_count} 个 3DGS`); if (!pieces.length) pieces.push("等待扫描"); return pieces.join(" · "); } function datasetPreviewLabel(item) { + if (item.scan?.spatial_bundle_ready) return "空间包"; if (item.source_type === "prepared_media") return "帧数据"; if (collectionDatasets().some((dataset) => dataset.id === item.id)) return "采集"; if ((item.scan?.video_count || 0) > 0) return "视频"; @@ -1173,6 +1203,8 @@ function taskTypeLabel(type) { reconstruction_preflight: "重建预检", scene_reconstruction: "场景重建", motrixsim_asset_import: "MotrixSim 资产导入", + motrixsim_scene_import: "MotrixSim 场景包导入", + scene_asset_compose: "数字资产合成", simulator_validation: "仿真校验", renderer_benchmark: "渲染器对比", scene_cover_render: "场景封面", @@ -1219,6 +1251,8 @@ function sourceTypeLabel(type) { directory: "目录", expert_collection: "专家采集数据", scene_reconstruction: "场景重建", + motrixsim_spatial_bundle: "MotrixSim 空间包", + scene_revision: "场景修订", training: "训练模型", inference: "推理结果", splat_or_pointcloud: "点云 / Splat", @@ -1362,7 +1396,7 @@ function exportSummary(payload = {}) { function switchTaskPage(type) { if (["hf_import", "media_prepare"].includes(type)) switchTab("data"); - else if (["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"].includes(type)) switchTab("reconstruct"); + else if (["scene_reconstruction", "reconstruction_preflight", "motrixsim_asset_import", "motrixsim_scene_import", "scene_asset_compose", "simulator_validation", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize"].includes(type)) switchTab("reconstruct"); else if (type === "expert_collection") switchTab("collect"); else if (type === "training") switchTab("train"); else if (type === "inference") switchTab("infer"); @@ -1561,6 +1595,7 @@ function bindEvents() { renderSceneList(); renderSceneViewerShell(); }); + $("assetPreset")?.addEventListener("change", updateDigitalAssetHint); $("reconstructInput")?.addEventListener("change", () => { renderReconstructDatasetCards(); }); @@ -1597,17 +1632,20 @@ function bindEvents() { const sceneId = $("assetScene")?.value || state.activeSceneId || state.scenes[0]?.id; if (!sceneId) return toast("请先选择一个场景。"); const payload = sceneAssetPayload(); - if (!payload.path) return toast("请选择资产来源,或在高级入口填写白名单文件。"); + if (!payload.primitive && !payload.source?.type) return toast("请先从数字资产库选择一个资产。"); const result = await api(`/api/scenes/${sceneId}/assets`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload), }); - state.activeSceneId = result.scene?.id || sceneId; - toast("资产已添加,并创建了新的场景修订版本"); + const task = result.task; + if (task) { + state.selectedTaskId = task.id; + state.tasks.unshift(task); + } + toast("资产合成任务已创建;双运行时校验通过后生成新版本"); await refresh({ quiet: true }); - renderSceneViewerShell(); - scrollIntoViewIfPossible("#assetRail"); + if (task) showTask(task.id).catch((error) => toast(error.message)); }); $("createCollectTask")?.addEventListener("click", async () => { @@ -1788,6 +1826,16 @@ async function handleBodyClick(event) { const preflightDataset = event.target.closest("[data-dataset-preflight]"); if (preflightDataset) await createTask("reconstruction_preflight", { dataset_id: preflightDataset.dataset.datasetPreflight, min_frames: Number($("sceneMinFrames")?.value || 30) }); + const motrixDataset = event.target.closest("[data-dataset-motrixsim]"); + if (motrixDataset) { + const task = await api(`/api/datasets/${motrixDataset.dataset.datasetMotrixsim}/import-motrixsim`, { method: "POST" }); + state.selectedTaskId = task.id; + toast("MotrixSim 场景包导入任务已创建"); + await refresh({ quiet: true }); + switchTaskPage("motrixsim_scene_import"); + showTask(task.id).catch((error) => toast(error.message)); + } + for (const kind of ["dataset", "scene", "model"]) { const edit = event.target.closest(`[data-edit-${kind}]`); if (edit) openEntityEditor(kind, edit.dataset[`edit${kind[0].toUpperCase()}${kind.slice(1)}`]); diff --git a/src/real2sim_platform/static/index.html b/src/real2sim_platform/static/index.html index 3d2f44c..d13dea3 100644 --- a/src/real2sim_platform/static/index.html +++ b/src/real2sim_platform/static/index.html @@ -5,7 +5,7 @@ Real2Sim - +
@@ -141,8 +141,8 @@ xarm7、3DGS fixture 和公开数据集 +
- -
- 高级:从白名单路径添加新资产 - -
- - - - + +
基础形状可直接加入,上传的网格会自动出现在资产库
+ + + + + + + + + + - + + + +
diff --git a/src/real2sim_platform/static/scene-viewer.html b/src/real2sim_platform/static/scene-viewer.html index 21d9195..0a6bef6 100644 --- a/src/real2sim_platform/static/scene-viewer.html +++ b/src/real2sim_platform/static/scene-viewer.html @@ -111,7 +111,15 @@ scene.add(light); scene.add(new THREE.AmbientLight(0x9fb6c8, 0.8)); + const fitRoot = new THREE.Group(); + const contentRoot = new THREE.Group(); + fitRoot.add(contentRoot); + scene.add(fitRoot); + let loadedObject = null; + let overlayObjects = []; + let manifestAssets = []; + let sourceUpAxis = "Z"; let sparkRenderer = null; let sparkModule = null; const debug = { @@ -119,6 +127,7 @@ loaded: false, assetName: "", pointCount: 0, + overlayCount: 0, errors: [], warnings: [], canvas: { width: 0, height: 0 }, @@ -193,6 +202,9 @@ return response.json(); }); const assets = Array.isArray(manifest.assets) ? prioritizeAssets(manifest.assets) : []; + manifestAssets = assets; + sourceUpAxis = String(manifest.coordinate_frame?.up_axis || "Z").toUpperCase(); + contentRoot.rotation.set(sourceUpAxis === "Z" ? -Math.PI / 2 : 0, 0, 0); const renderable = assets.filter(isRenderableAsset); assetList.innerHTML = assets.slice(0, 20).map((item, index) => { const quality = assetQuality(item, index); @@ -208,7 +220,12 @@ showAssetInfo(assets[Number(button.dataset.index)]); return; } - loadAsset(assets[Number(button.dataset.index)], { manual: true }).catch(showError); + const selected = assets[Number(button.dataset.index)]; + if (selected?.role === "added_asset") { + status.textContent = `已添加资产 · ${selected.name || "asset"} · 与 MotrixSim 场景使用同一坐标`; + return; + } + loadAsset(selected, { manual: true }).then(() => loadSceneOverlays(assets)).catch(showError); }); status.textContent = rendererStatusText(manifest, assets.length); hint.textContent = viewerHint(assets, renderable); @@ -221,6 +238,7 @@ // Large 3DGS assets remain an explicit user action. Their real cover is // a safer first frame than an automatically generated diagnostic mesh. await loadAsset(preferred, { manual: preferred === previewImage }); + await loadSceneOverlays(assets); } else if (renderable[0]) addPlaceholder(`可渲染大文件未自动加载:${renderable[0].name || "asset"} · ${renderable[0].size_label || "unknown size"} · 点击按钮手动加载`); else addPlaceholder("当前清单没有可直接渲染的 PLY/SPZ/SPLAT/GLB/OBJ 资产"); @@ -264,7 +282,8 @@ addPlaceholder(`large asset skipped: ${asset.name || "asset"} · ${asset.size_label || "unknown size"} · click to load`); return; } - if (loadedObject) scene.remove(loadedObject); + if (loadedObject) contentRoot.remove(loadedObject); + clearOverlays(); const suffix = (asset.name || asset.path || "").toLowerCase().split(".").pop(); const imageAsset = ["png", "jpg", "jpeg", "webp"].includes(suffix); if (!imageAsset) prepareGeometryScene(); @@ -284,16 +303,16 @@ const { GLTFLoader } = await import("three/addons/loaders/GLTFLoader.js"); const gltf = await withLoadingTimeout(new GLTFLoader().loadAsync(asset.url), `loading timed out: ${asset.name || "asset"}${size}`); loadedObject = gltf.scene; + contentRoot.add(loadedObject); normalizeObjectFromBox(loadedObject); - scene.add(loadedObject); debug.backend = "three_fallback"; debug.loaded = true; status.textContent = `Three.js fallback · ${asset.name}`; } else if (suffix === "obj") { status.textContent = `Three.js fallback · loading ${asset.name}${size}`; loadedObject = await withLoadingTimeout(new OBJLoader().loadAsync(asset.url), `loading timed out: ${asset.name || "asset"}${size}`); + contentRoot.add(loadedObject); normalizeObjectFromBox(loadedObject); - scene.add(loadedObject); debug.backend = "three_fallback"; debug.loaded = true; status.textContent = `Three.js fallback · ${asset.name}`; @@ -309,11 +328,13 @@ } function isRenderableAsset(asset) { + if (asset?.role === "added_asset" && asset?.viewer_primitive) return true; const suffix = String(asset?.name || asset?.path || "").toLowerCase().split(".").pop(); return ["ply", "spz", "splat", "ksplat", "sog", "glb", "gltf", "obj", "png", "jpg", "jpeg", "webp"].includes(suffix); } function assetTypeLabel(asset) { + if (asset?.role === "added_asset") return "已添加资产"; const suffix = String(asset?.name || asset?.path || "").toLowerCase().split(".").pop(); const kind = String(asset?.kind || ""); if (["ply", "spz", "splat", "ksplat", "sog", "glb", "gltf", "obj"].includes(suffix)) return "可预览"; @@ -351,7 +372,8 @@ } async function addImagePreview(asset) { - if (loadedObject) scene.remove(loadedObject); + if (loadedObject) contentRoot.remove(loadedObject); + clearOverlays(); loadedObject = null; status.textContent = `图片预览 · loading ${asset.name || "asset"}`; const texture = await withLoadingTimeout(new THREE.TextureLoader().loadAsync(asset.url), `loading timed out: ${asset.name || "asset"}`); @@ -420,7 +442,7 @@ status.textContent = `Spark · loading ${asset.name}${asset.size_label ? ` · ${asset.size_label}` : ""}`; const splatMesh = new spark.SplatMesh({ url: asset.url }); loadedObject = splatMesh; - scene.add(splatMesh); + contentRoot.add(splatMesh); await splatMesh.initialized; normalizeObjectFromBox(splatMesh); const count = splatMesh.splats?.getNumSplats?.() || splatMesh.packedSplats?.numSplats || 0; @@ -432,7 +454,7 @@ console.info("Spark preview unavailable; using Three.js fallback", error); debug.warnings.push(error.message || String(error)); if (loadedObject) { - scene.remove(loadedObject); + contentRoot.remove(loadedObject); loadedObject = null; } if (suffix !== "ply") throw error; @@ -453,14 +475,98 @@ color: 0x58c4a8, }); loadedObject = new THREE.Points(geometry, material); + contentRoot.add(loadedObject); normalizeObject(loadedObject, geometry.boundingSphere); - scene.add(loadedObject); debug.backend = "three_fallback"; debug.loaded = true; debug.pointCount = count; status.textContent = `Three.js fallback · ${asset.name} · ${count} points`; } + async function loadSceneOverlays(assets) { + clearOverlays(); + if (!loadedObject) return; + const overlays = assets.filter((asset) => asset?.role === "added_asset"); + for (const asset of overlays) { + try { + const object = await createOverlayObject(asset); + if (!object) continue; + applyAssetTransform(object, asset.transform || {}); + object.userData.real2simAsset = asset.name || asset.path || "asset"; + contentRoot.add(object); + overlayObjects.push(object); + } catch (error) { + debug.warnings.push(`overlay ${asset.name || "asset"}: ${error.message || error}`); + console.warn("Unable to load scene overlay", asset, error); + } + } + debug.overlayCount = overlayObjects.length; + if (overlayObjects.length) hint.textContent = `已叠加 ${overlayObjects.length} 个 MotrixSim 数字资产;位置、旋转、缩放与仿真场景保持一致。`; + } + + async function createOverlayObject(asset) { + const primitive = String(asset.viewer_primitive || "").toLowerCase(); + const dimensions = Array.isArray(asset.physics?.dimensions) ? asset.physics.dimensions.map(Number) : [0.1, 0.1, 0.1]; + const rgba = Array.isArray(asset.physics?.rgba) ? asset.physics.rgba.map(Number) : [0.13, 0.76, 0.67, 1]; + const material = new THREE.MeshStandardMaterial({ + color: new THREE.Color(rgba[0], rgba[1], rgba[2]), + transparent: rgba[3] < 1, + opacity: rgba[3], + roughness: 0.55, + metalness: 0.05, + }); + if (primitive === "box") return new THREE.Mesh(new THREE.BoxGeometry(dimensions[0], dimensions[1], dimensions[2]), material); + if (primitive === "sphere") return new THREE.Mesh(new THREE.SphereGeometry(Math.max(...dimensions) / 2, 32, 20), material); + if (primitive === "cylinder") { + const geometry = new THREE.CylinderGeometry(Math.max(dimensions[0], dimensions[1]) / 2, Math.max(dimensions[0], dimensions[1]) / 2, dimensions[2], 32); + geometry.rotateX(Math.PI / 2); + return new THREE.Mesh(geometry, material); + } + if (primitive === "capsule") { + const radius = Math.max(dimensions[0], dimensions[1]) / 2; + const bodyLength = Math.max(0.001, dimensions[2] - radius * 2); + const group = new THREE.Group(); + const cylinder = new THREE.CylinderGeometry(radius, radius, bodyLength, 32); + cylinder.rotateX(Math.PI / 2); + group.add(new THREE.Mesh(cylinder, material)); + const capGeometry = new THREE.SphereGeometry(radius, 28, 18); + const first = new THREE.Mesh(capGeometry, material); + const second = new THREE.Mesh(capGeometry, material); + first.position.z = bodyLength / 2; + second.position.z = -bodyLength / 2; + group.add(first, second); + return group; + } + if (!asset.url) return null; + const suffix = String(asset.name || asset.path || "").toLowerCase().split(".").pop(); + if (suffix === "glb" || suffix === "gltf") { + const { GLTFLoader } = await import("three/addons/loaders/GLTFLoader.js"); + return (await new GLTFLoader().loadAsync(asset.url)).scene; + } + if (suffix === "obj") return new OBJLoader().loadAsync(asset.url); + if (suffix === "ply") { + const geometry = await new PLYLoader().loadAsync(asset.url); + geometry.computeVertexNormals(); + return new THREE.Mesh(geometry, material); + } + return null; + } + + function applyAssetTransform(object, transform) { + const translation = Array.isArray(transform.translation) ? transform.translation.map(Number) : [0, 0, 0]; + const rotation = Array.isArray(transform.rotation_euler_degrees) ? transform.rotation_euler_degrees.map(Number) : [0, 0, Number(transform.yaw_degrees || 0)]; + const scale = Array.isArray(transform.scale) ? transform.scale.map(Number) : [Number(transform.scale || 1), Number(transform.scale || 1), Number(transform.scale || 1)]; + object.position.set(translation[0] || 0, translation[1] || 0, translation[2] || 0); + object.rotation.set(THREE.MathUtils.degToRad(rotation[0] || 0), THREE.MathUtils.degToRad(rotation[1] || 0), THREE.MathUtils.degToRad(rotation[2] || 0)); + object.scale.set(scale[0] || 1, scale[1] || 1, scale[2] || 1); + } + + function clearOverlays() { + for (const object of overlayObjects) contentRoot.remove(object); + overlayObjects = []; + debug.overlayCount = 0; + } + function pointSizeFor(count, asset) { const name = String(asset?.name || asset?.path || "").toLowerCase(); if (name.includes("colmap_sparse") || name.includes("sparse_points")) return 0.022; @@ -486,11 +592,17 @@ return sparkModule; } - function normalizeObject(object, sphere) { - if (!sphere) return; - object.position.sub(sphere.center); + function normalizeObject(object, _sphere) { + fitRoot.position.set(0, 0, 0); + fitRoot.scale.set(1, 1, 1); + object.updateMatrixWorld(true); + const box = new THREE.Box3().setFromObject(object); + if (!Number.isFinite(box.min.x) || !Number.isFinite(box.max.x)) return; + const sphere = new THREE.Sphere(); + box.getBoundingSphere(sphere); const scale = sphere.radius > 0 ? 1.6 / sphere.radius : 1; - object.scale.setScalar(scale); + fitRoot.scale.setScalar(scale); + fitRoot.position.copy(sphere.center).multiplyScalar(-scale); controls.target.set(0, 0, 0); setCoverCamera(); controls.update(); @@ -512,19 +624,17 @@ } function normalizeObjectFromBox(object) { - const box = new THREE.Box3().setFromObject(object); - if (!Number.isFinite(box.min.x) || !Number.isFinite(box.max.x)) return; - const sphere = new THREE.Sphere(); - box.getBoundingSphere(sphere); - normalizeObject(object, sphere); + normalizeObject(object, null); } function addPlaceholder(message) { - if (loadedObject) scene.remove(loadedObject); + if (loadedObject) contentRoot.remove(loadedObject); + clearOverlays(); const geometry = new THREE.BoxGeometry(0.8, 0.8, 0.8); const material = new THREE.MeshStandardMaterial({ color: 0x58c4a8, roughness: 0.65 }); loadedObject = new THREE.Mesh(geometry, material); - scene.add(loadedObject); + contentRoot.add(loadedObject); + normalizeObjectFromBox(loadedObject); debug.backend = "placeholder"; debug.loaded = false; status.textContent = message; diff --git a/src/real2sim_platform/static/styles.css b/src/real2sim_platform/static/styles.css index e86cccc..6ff1566 100644 --- a/src/real2sim_platform/static/styles.css +++ b/src/real2sim_platform/static/styles.css @@ -220,6 +220,11 @@ body.viewer-maximized { overflow: hidden; } .asset-grid, .model-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 8px; padding-top: 11px; } .asset-card, .model-card { min-width: 0; min-height: 128px; display: grid; align-content: end; gap: 6px; padding: 10px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } .asset-card span, .model-card > span, .model-card p { margin: 0; color: var(--muted); font-size: 10px; } +.asset-source-actions { min-height: 42px; display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 7px 9px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.asset-source-actions span { min-width: 0; color: var(--muted); font-size: 10px; line-height: 1.45; } +.asset-source-actions button { flex: 0 0 auto; } +.toggle-field { min-height: 42px; display: flex !important; grid-template-columns: auto 1fr !important; align-items: center; justify-content: start; gap: 8px !important; padding: 0 2px; } +.toggle-field input { width: 16px; height: 16px; margin: 0; } .model-card { min-height: 160px; align-content: start; } .sparkline { height: 34px; display: flex; align-items: end; border-bottom: 1px solid var(--line-soft); } .sparkline i { height: 5px; border-radius: 4px; background: var(--accent); } diff --git a/tests/test_platform.py b/tests/test_platform.py index d9d213b..8026e1e 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -193,37 +193,38 @@ def test_dataset_scene_model_bulk_interfaces(tmp_path: Path) -> None: assert model.json()["type"] == "training" -def test_scene_asset_append_updates_manifest_and_artifact(tmp_path: Path) -> None: +def test_scene_asset_request_queues_real_revision_task_from_catalog(tmp_path: Path) -> None: package = tmp_path / "scene_package" package.mkdir() scene_file = package / "scene.json" scene_file.write_text('{"schema":"motrixsim.scene.v1"}\n', encoding="utf-8") - asset = package / "added_asset.json" - asset.write_text('{"name":"fixture asset"}\n', encoding="utf-8") + asset_dataset = tmp_path / "asset_dataset" + asset_dataset.mkdir() + asset = asset_dataset / "fixture.obj" + asset.write_text("v 0 0 0\nv 0.1 0 0\nv 0 0.1 0\nf 1 2 3\n", encoding="utf-8") cfg = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", allowed_path_roots=(tmp_path,)) client = TestClient(create_app(cfg)) + dataset = client.post("/api/datasets/import-path", json={"path": str(asset_dataset), "name": "asset library"}).json() scene = client.post("/api/scenes/import", json={"path": str(scene_file), "name": "editable scene"}).json() + catalog = client.get("/api/assets").json() + catalog_asset = next(item for item in catalog if item.get("relative_path") == "fixture.obj") + assert catalog_asset["source"]["dataset_id"] == dataset["id"] response = client.post( f"/api/scenes/{scene['id']}/assets", json={ - "path": str(asset), - "name": "fixture metadata", - "role": "simulator_import", - "transform": {"translation": [1, 2, 3], "scale": 1.5}, + "source": catalog_asset["source"], + "name": "fixture mesh", + "transform": {"translation": [1, 2, 3], "scale": [1.5, 1.5, 1.5]}, + "physics": {"dynamic": True, "mass": 0.2, "collision_mode": "visual_and_collision"}, }, ) assert response.status_code == 200, response.text payload = response.json() - assert payload["asset"]["name"] == "fixture metadata" - assert payload["scene"]["id"] != scene["id"] - assert payload["parent_scene_id"] == scene["id"] - manifest = client.get(f"/api/scenes/{payload['scene']['id']}/manifest").json() - added = [item for item in manifest["assets"] if item["name"] == "fixture metadata"] - assert added - assert added[0]["role"] == "simulator_import" - assert added[0]["transform"]["translation"] == [1, 2, 3] - assert manifest["revision"]["parent_scene_id"] == scene["id"] - assert client.get(added[0]["url"]).status_code == 200 + assert payload["scene_id"] == scene["id"] + assert payload["task"]["type"] == "scene_asset_compose" + assert payload["task"]["params"]["source"] == catalog_asset["source"] + assert payload["task"]["params"]["physics"]["dynamic"] is True + assert len(client.get("/api/scenes").json()) == 1 def test_scene_runtime_metadata_update(tmp_path: Path) -> None: diff --git a/tests/test_production_platform.py b/tests/test_production_platform.py index a12a591..ff43d09 100644 --- a/tests/test_production_platform.py +++ b/tests/test_production_platform.py @@ -13,6 +13,17 @@ from fastapi.testclient import TestClient from real2sim_platform.config import PlatformConfig from real2sim_platform.adapters import _act_task_config_source, _inspect_act_dataset, index_artifacts_for_task +from real2sim_platform.motrix_scene import ( + compose_scene_asset, + convert_asset_to_obj, + detect_spatial_bundle, + export_combined_obj, + export_glb, + load_usd_meshes, + safe_extract_zip, + validate_runtime_scene, + write_runtime_scene, +) from real2sim_platform.rm65_controller import ACTION_ALIGNMENT, CONTROL_HOLD_STEPS, _write_preview_video, collect from real2sim_platform.runtime_gate import _motrixsim_assets_probe, _spz_exporter_probe, runtime_key from real2sim_platform.server import create_app @@ -24,6 +35,138 @@ from real2sim_platform.store import PlatformStore ROOT = Path(__file__).resolve().parents[1] +def test_spatial_bundle_detection_and_safe_archive_boundary(tmp_path: Path) -> None: + import zipfile + + bundle = tmp_path / "bundle" + (bundle / "combined_world" / "usdz").mkdir(parents=True) + (bundle / "gaussian_splatting").mkdir() + (bundle / "combined_world" / "usdz" / "environment_collider.usdz").write_bytes(b"collider") + (bundle / "combined_world" / "usdz" / "scene_shaded.usdz").write_bytes(b"visual") + (bundle / "gaussian_splatting" / "full_res.spz").write_bytes(b"spz") + detected = detect_spatial_bundle(bundle) + assert detected["ready"] is True + assert detected["layers"]["collision"]["sha256"] + archive_path = tmp_path / "unsafe.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("../outside.txt", "not allowed") + with zipfile.ZipFile(archive_path) as archive, pytest.raises(ValueError, match="unsafe archive path"): + safe_extract_zip(archive, tmp_path / "out") + + +def test_real_usd_conversion_exports_motrixsim_mesh_and_mujoco_loads(tmp_path: Path) -> None: + pytest.importorskip("pxr") + pytest.importorskip("trimesh") + pytest.importorskip("mujoco") + usd = tmp_path / "fixture.usda" + usd.write_text( + '''#usda 1.0 +( + defaultPrim = "World" + metersPerUnit = 1 + upAxis = "Z" +) +def Xform "World" { + def Mesh "Floor" { + point3f[] points = [(-1, -1, 0), (1, -1, 0), (1, 1, 0), (-1, 1, 0), (-1, -1, -0.1), (1, -1, -0.1), (1, 1, -0.1), (-1, 1, -0.1)] + int[] faceVertexCounts = [4, 4, 4, 4, 4, 4] + int[] faceVertexIndices = [0, 1, 2, 3, 7, 6, 5, 4, 0, 4, 5, 1, 1, 5, 6, 2, 2, 6, 7, 3, 3, 7, 4, 0] + uniform token subdivisionScheme = "none" + } +} +''', + encoding="utf-8", + ) + meshes, metadata = load_usd_meshes(usd) + assert metadata["up_axis"] == "Z" + assert metadata["vertices"] == 8 + assert metadata["triangles"] == 12 + mesh_dir = tmp_path / "package" / "assets" / "meshes" + collision = mesh_dir / "environment_collision.obj" + visual = mesh_dir / "environment_visual.obj" + assert export_combined_obj(meshes, collision)["triangles"] == 12 + assert export_combined_obj(meshes, visual)["vertices"] == 8 + glb = tmp_path / "package" / "viewer" / "environment.glb" + assert export_glb(meshes, glb)["size_bytes"] > 0 + runtime = tmp_path / "package" / "simulator_assets" / "scene.xml" + write_runtime_scene(runtime, scene_name="USD fixture", collision_obj=collision, visual_obj=visual) + report = validate_runtime_scene(runtime, steps=8) + assert report["mujoco"]["ok"] is True, report + + +def test_glb_asset_conversion_bakes_scene_node_transforms(tmp_path: Path) -> None: + trimesh = pytest.importorskip("trimesh") + source = tmp_path / "translated.glb" + scene = trimesh.Scene() + scene.add_geometry( + trimesh.creation.box(extents=[1.0, 1.0, 1.0]), + node_name="translated_box", + transform=trimesh.transformations.translation_matrix([4.0, 2.0, 1.0]), + ) + scene.export(source) + output = tmp_path / "translated.obj" + convert_asset_to_obj(source, output) + converted = trimesh.load(output, force="mesh", process=False) + assert converted.bounds[0].tolist() == pytest.approx([3.5, 1.5, 0.5]) + assert converted.bounds[1].tolist() == pytest.approx([4.5, 2.5, 1.5]) + + +def test_primitive_asset_scale_is_applied_to_runtime_geometry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = tmp_path / "scene-package" + mesh_dir = package / "assets" / "meshes" + mesh_dir.mkdir(parents=True) + collision = mesh_dir / "collision.obj" + visual = mesh_dir / "visual.obj" + cube = { + "name": "cube", + "vertices": np.asarray( + [[-1, -1, -1], [1, -1, -1], [1, 1, -1], [-1, 1, -1], [-1, -1, 1], [1, -1, 1], [1, 1, 1], [-1, 1, 1]], + dtype=float, + ), + "faces": np.asarray( + [[0, 1, 2], [0, 2, 3], [4, 6, 5], [4, 7, 6], [0, 4, 5], [0, 5, 1], [1, 5, 6], [1, 6, 2], [2, 6, 7], [2, 7, 3], [3, 7, 4], [3, 4, 0]], + dtype=int, + ), + } + export_combined_obj([cube], collision) + export_combined_obj([cube], visual) + runtime = package / "simulator_assets" / "scene.xml" + write_runtime_scene(runtime, scene_name="scale fixture", collision_obj=collision, visual_obj=visual) + (package / "scene_manifest.json").write_text( + json.dumps({"assets": [], "simulator_artifacts": {"motrixsim": {"runtime_scene": str(runtime)}}}), + encoding="utf-8", + ) + monkeypatch.setattr( + "real2sim_platform.motrix_scene.validate_runtime_scene", + lambda _path, steps: { + "ok": True, + "steps": steps, + "mujoco": {"ok": True}, + "motrixsim": {"ok": True}, + }, + ) + compose_scene_asset( + package, + asset_spec={ + "name": "scaled box", + "primitive": "box", + "transform": {"scale": [2.0, 3.0, 4.0]}, + "physics": {"dimensions": [1.0, 2.0, 3.0], "dynamic": True}, + }, + source_path=None, + validation_steps=4, + ) + from xml.etree import ElementTree as ET + + bodies = ET.parse(runtime).getroot().findall("./worldbody/body") + geom = bodies[-1].find("geom") + assert geom is not None + assert [float(value) for value in geom.attrib["size"].split()] == pytest.approx([1.0, 3.0, 6.0]) + + def test_ascii_3dgs_spz_exporter_uses_real_spark_writer(tmp_path: Path) -> None: config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace") probe = _spz_exporter_probe(config)