3139 lines
146 KiB
Python
3139 lines
146 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import hashlib
|
||
from functools import lru_cache
|
||
import math
|
||
import os
|
||
import shutil
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
from typing import Any, Callable
|
||
from urllib.parse import quote, unquote, urlparse
|
||
from urllib.request import urlopen
|
||
|
||
from .config import PlatformConfig
|
||
from .source_revision import read_source_revision
|
||
from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path
|
||
from .store import PlatformStore
|
||
|
||
|
||
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".ppm"}
|
||
VIDEO_SUFFIXES = {".mp4", ".mov", ".m4v"}
|
||
PARQUET_SUFFIXES = {".parquet"}
|
||
SCENE_ASSET_SUFFIXES = {".ply", ".splat", ".spz", ".ksplat", ".sog", ".zip", ".rad", ".obj", ".glb", ".gltf", ".xml", ".json", ".png", ".jpg", ".jpeg"}
|
||
SPARK_ASSET_SUFFIXES = {".ply", ".splat", ".spz", ".ksplat", ".sog", ".zip", ".rad"}
|
||
SPARK_VENDOR_PATH = Path(__file__).parent / "static" / "vendor" / "spark.module.min.js"
|
||
THREE_VENDOR_PATH = Path(__file__).parent / "static" / "vendor" / "three" / "three.module.js"
|
||
VIEWER_AUTOLOAD_MAX_BYTES = 25 * 1024 * 1024
|
||
MEDIA_PREPARE_DEFAULT_CAMERA = "observation.images.front"
|
||
MEDIA_PREPARE_DEFAULT_SAMPLE_FPS = 2.0
|
||
MEDIA_PREPARE_DEFAULT_MAX_FRAMES = 80
|
||
RECONSTRUCTION_DEFAULT_MIN_FRAMES = 30
|
||
RECONSTRUCTION_DEFAULT_TARGET_FRAMES = 80
|
||
SIM_VALIDATION_LEVELS = {"smoke", "runtime_if_available", "runtime_required"}
|
||
|
||
|
||
class TaskCanceled(RuntimeError):
|
||
pass
|
||
|
||
|
||
def scan_dataset(path: Path) -> dict[str, Any]:
|
||
root = path.resolve()
|
||
files = [root] if root.is_file() else [p for p in root.rglob("*") if p.is_file()]
|
||
images = [p for p in files if p.suffix.lower() in IMAGE_SUFFIXES and "depth" not in {part.lower() for part in p.parts}]
|
||
depths = [p for p in files if p.suffix.lower() in {".exr", ".npy", ".png"} and "depth" in {part.lower() for part in p.parts}]
|
||
videos = [p for p in files if p.suffix.lower() in VIDEO_SUFFIXES]
|
||
parquets = [p for p in files if p.suffix.lower() in PARQUET_SUFFIXES]
|
||
metadata = [p for p in files if p.name in {"metadata.json", "transforms.json", "calibration.json"} or p.suffix.lower() == ".jsonl"]
|
||
prepared_manifest = root / "prepared_manifest.json" if root.is_dir() else None
|
||
input_kind = "prepared_frames" if prepared_manifest and prepared_manifest.is_file() else "video" if videos or root.suffix.lower() in VIDEO_SUFFIXES else "parquet_video" if parquets else "rgbd" if depths else "images"
|
||
return {
|
||
"path": str(root),
|
||
"image_count": len(images),
|
||
"depth_count": len(depths),
|
||
"video_count": len(videos),
|
||
"parquet_count": len(parquets),
|
||
"metadata_files": [str(p.relative_to(root)) if root.is_dir() else p.name for p in metadata[:20]],
|
||
"prepared_manifest": str(prepared_manifest) if prepared_manifest and prepared_manifest.is_file() else "",
|
||
"input_kind": input_kind,
|
||
}
|
||
|
||
|
||
def run_doctor(config: PlatformConfig) -> dict[str, Any]:
|
||
core_checks = (
|
||
[
|
||
"workspace_writable",
|
||
"rm65_controller",
|
||
"discoverse_act",
|
||
"torch_cuda",
|
||
"act_import",
|
||
"media_prepare_deps",
|
||
"ffmpeg",
|
||
"colmap",
|
||
"gs_video_reconstruct",
|
||
"anysplat_pinned",
|
||
"mujoco_python",
|
||
"motrixsim_python",
|
||
"nvidia_smi",
|
||
]
|
||
if config.production_mode
|
||
else ["workspace_writable", "expert_policy", "training", "inference", "media_prepare_deps", "ffmpeg", "spark_bundle", "three_bundle"]
|
||
)
|
||
act_root = config.discoverse_root / "policies" / "act"
|
||
checks = {
|
||
"workspace_writable": _is_writable(config.workspace_dir),
|
||
"expert_policy": (config.expert_policy_root / "src/real2sim_expert/collect.py").is_file(),
|
||
"training": (config.training_root / "src/real2sim_train/train.py").is_file(),
|
||
"inference": (config.inference_root / "src/real2sim_infer/run.py").is_file(),
|
||
"gs_video_reconstruct": (config.gs_playground_root / "scripts/real2sim_video_motrix_scene.py").is_file(),
|
||
"images23dgs": (config.images23dgs_root / "images23dgs/product/server.py").is_file(),
|
||
"colmap": shutil.which(str(config.colmap_binary)) is not None or config.colmap_binary.exists(),
|
||
"gsplat_python": config.gsplat_python.exists(),
|
||
"anysplat_root": config.anysplat_root.is_dir(),
|
||
"anysplat_python": config.anysplat_python.exists(),
|
||
"runtime_env_script": config.runtime_env_script.is_file(),
|
||
"cuda_home": config.cuda_home.is_dir(),
|
||
"media_prepare_deps": all(_module_available(name) for name in ["huggingface_hub", "pyarrow", "PIL", "imageio", "cv2"]),
|
||
"ffmpeg": shutil.which("ffmpeg") is not None,
|
||
"spark_bundle": SPARK_VENDOR_PATH.is_file(),
|
||
"three_bundle": THREE_VENDOR_PATH.is_file(),
|
||
"mujoco_python": _module_available("mujoco"),
|
||
"service_process": os.getpid() > 0,
|
||
"nvidia_smi": shutil.which("nvidia-smi") is not None,
|
||
"rm65_controller": Path(__file__).with_name("rm65_controller.py").is_file(),
|
||
"discoverse_act": _git_head(config.discoverse_root) == config.discoverse_commit and _git_head(act_root) == config.act_commit,
|
||
"torch_cuda": _torch_cuda_ready(),
|
||
"act_import": _act_runtime_import_ready(str(config.discoverse_root), config.discoverse_commit, config.act_commit),
|
||
"anysplat_pinned": _git_head(config.anysplat_root) == config.anysplat_commit and config.anysplat_python.exists(),
|
||
"motrixsim_python": _module_available("motrixsim"),
|
||
}
|
||
return {
|
||
"ok": all(bool(checks[name]) for name in core_checks),
|
||
"core_checks": core_checks,
|
||
"warnings": [
|
||
name
|
||
for name in [
|
||
"images23dgs",
|
||
"mujoco_python",
|
||
"gs_video_reconstruct",
|
||
"gsplat_python",
|
||
"anysplat_root",
|
||
"anysplat_python",
|
||
"runtime_env_script",
|
||
"cuda_home",
|
||
"colmap",
|
||
"nvidia_smi",
|
||
"torch_cuda",
|
||
"act_import",
|
||
"three_bundle",
|
||
]
|
||
if not checks.get(name)
|
||
],
|
||
"checks": checks,
|
||
"paths": {
|
||
"workspace": str(config.workspace_dir),
|
||
"gs_playground_root": str(config.gs_playground_root),
|
||
"images23dgs_root": str(config.images23dgs_root),
|
||
"colmap_binary": str(config.colmap_binary),
|
||
"gsplat_python": str(config.gsplat_python),
|
||
"anysplat_root": str(config.anysplat_root),
|
||
"anysplat_python": str(config.anysplat_python),
|
||
"runtime_root": str(config.runtime_root),
|
||
"runtime_env_script": str(config.runtime_env_script),
|
||
"cuda_home": str(config.cuda_home),
|
||
"spark_bundle": str(SPARK_VENDOR_PATH),
|
||
"three_bundle": str(THREE_VENDOR_PATH),
|
||
},
|
||
"renderers": renderer_decision(),
|
||
"simulators": ["mujoco", "motrixsim"],
|
||
}
|
||
|
||
|
||
def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, Any]) -> dict[str, Any]:
|
||
run_dir = Path(task["run_dir"])
|
||
log_path = run_dir / "logs" / "task.log"
|
||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||
log_path.write_text(f"task {task['id']} type={task['type']} started\n", encoding="utf-8")
|
||
emit = lambda message: _append_log(log_path, message)
|
||
_task_progress(store, task, 8, "准备执行", "正在校验任务参数和运行目录")
|
||
task_type = task["type"]
|
||
if task_type == "scene_reconstruction":
|
||
return run_scene_reconstruction(config, store, task, emit)
|
||
if task_type == "media_prepare":
|
||
return run_media_prepare(config, store, task, emit)
|
||
if task_type == "reconstruction_preflight":
|
||
return run_reconstruction_preflight(config, store, task, emit)
|
||
if task_type == "hf_import":
|
||
return run_hf_import(config, store, task, emit)
|
||
if task_type == "bos_import":
|
||
return run_bos_import(config, store, task, emit)
|
||
if task_type == "motrixsim_asset_import":
|
||
return run_motrixsim_asset_import(config, store, task, emit)
|
||
if task_type == "simulator_validation":
|
||
return run_simulator_validation(config, store, task, emit)
|
||
if task_type == "expert_collection":
|
||
return run_expert_collection(config, store, task, emit)
|
||
if task_type == "training":
|
||
return run_training(config, store, task, emit)
|
||
if task_type == "inference":
|
||
return run_inference(config, store, task, emit)
|
||
if task_type == "renderer_benchmark":
|
||
return run_renderer_benchmark(config, store, task, emit)
|
||
if task_type == "scene_cover_render":
|
||
return run_scene_cover_render(config, store, task, emit)
|
||
if task_type == "scene_splat_optimize":
|
||
return run_scene_splat_optimize(config, store, task, emit)
|
||
raise ValueError(f"unknown task type: {task_type}")
|
||
|
||
|
||
def sync_task_outputs(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]:
|
||
storage = BosStorage(config)
|
||
if not storage.configured:
|
||
if config.production_mode:
|
||
raise RuntimeError("生产任务产物无法持久化:BOS 尚未配置")
|
||
return result
|
||
_task_progress(store, task, 96, "持久化产物", "正在将任务报告和业务产物上传到 BOS")
|
||
run_dir = Path(task["run_dir"])
|
||
if run_dir.is_dir():
|
||
_upload_directory(storage, run_dir, area="runs", relative_root=task["id"])
|
||
resource_specs = [
|
||
("dataset_id", "datasets", store.get_dataset),
|
||
("scene_id", "scenes", store.get_scene),
|
||
("model_id", "models", store.get_model),
|
||
]
|
||
persisted: dict[str, Any] = {}
|
||
for result_key, area, getter in resource_specs:
|
||
entity_id = str(result.get(result_key) or "")
|
||
if not entity_id:
|
||
continue
|
||
entity = getter(entity_id)
|
||
path = Path(str(entity["path"]))
|
||
root = path if path.is_dir() else path.parent
|
||
if not root.is_dir():
|
||
raise FileNotFoundError(f"resource output directory not found: {root}")
|
||
_upload_directory(storage, root, area=area, relative_root=entity_id)
|
||
checksum = _directory_digest(root)
|
||
uri = storage.uri_for_key(storage.object_key(area, f"{entity_id}/"))
|
||
store.update_entity_storage(area, entity_id, storage_uri=uri, checksum=checksum, cache_state="local")
|
||
persisted[result_key] = {"storage_uri": uri, "checksum": checksum}
|
||
result = dict(result)
|
||
result["persistent_outputs"] = persisted
|
||
result["run_storage_uri"] = storage.uri_for_key(storage.object_key("runs", f"{task['id']}/"))
|
||
return result
|
||
|
||
|
||
def _task_progress(store: PlatformStore, task: dict[str, Any] | str, progress: int, stage: str, message: str = "") -> None:
|
||
task_id = task if isinstance(task, str) else task["id"]
|
||
store.update_task(task_id, progress=progress, stage=stage, message=message)
|
||
|
||
|
||
def run_scene_reconstruction(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
package_dir = run_dir / "scene_package"
|
||
work_dir = run_dir / "work"
|
||
dry_run = bool(params.get("dry_run", False))
|
||
_task_progress(store, task, 12, "解析输入", "正在解析数据集和重建参数")
|
||
requested_input = _task_input_path(store, params)
|
||
input_path = requested_input if dry_run else _resolve_reconstruction_input(config, store, task, emit)
|
||
renderer = _select_renderer(params.get("renderer", config.default_renderer), package_dir)
|
||
simulators = params.get("simulators", ["mujoco", "motrixsim"])
|
||
if dry_run:
|
||
_task_progress(store, task, 35, "生成样例场景", "正在创建可预览场景包")
|
||
external_assets = _add_preview_asset_artifacts(store, task["id"], input_path)
|
||
scene_path = _write_dry_run_scene(package_dir, input_path, renderer, simulators)
|
||
manifest_path = _write_scene_manifest(package_dir, input_path=input_path, renderer=renderer, simulators=simulators, dry_run=True, external_assets=external_assets)
|
||
scene = store.create_scene(
|
||
name=params.get("scene_name", "dry-run scene"),
|
||
path=scene_path,
|
||
metadata={"renderer": renderer, "simulators": simulators, "dry_run": True, "manifest_path": str(manifest_path)},
|
||
source_type="scene_reconstruction",
|
||
source_uri=str(input_path),
|
||
parent_task_id=task["id"],
|
||
manifest_path=manifest_path,
|
||
tags=["dry_run", renderer, *simulators],
|
||
)
|
||
_index_artifacts(store, task["id"], package_dir)
|
||
_task_progress(store, task, 95, "索引产物", "场景包和预览产物已写入")
|
||
return {"scene_id": scene["id"], "scene_path": str(scene_path), "manifest_path": str(manifest_path), "dry_run": True, "renderer": renderer, "simulators": simulators}
|
||
|
||
script = config.gs_playground_root / "scripts/real2sim_video_motrix_scene.py"
|
||
if not script.is_file():
|
||
raise FileNotFoundError(script)
|
||
work_dir.mkdir(parents=True, exist_ok=True)
|
||
_task_progress(store, task, 22, "重建预检", "正在检查帧数、清晰度、视角变化和运行时")
|
||
preflight = build_reconstruction_preflight_report(config, store, params=params, input_path=input_path, run_dir=work_dir, emit=emit)
|
||
preflight_required = bool(params.get("preflight_required", True))
|
||
if preflight_required and not preflight.get("ok") and not bool(params.get("allow_low_quality", False)):
|
||
readable = preflight.get("readable_error") or "reconstruction preflight failed"
|
||
acceptance = {
|
||
"schema": "real2sim.capture_acceptance.v1",
|
||
"status": "not_run",
|
||
"reason": readable,
|
||
"preflight_report": str(work_dir / "capture_preflight_report.json"),
|
||
}
|
||
(work_dir / "capture_acceptance_report.json").write_text(json.dumps(acceptance, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
report = {
|
||
"schema": "real2sim.reconstruction_report.v1",
|
||
"status": "failed",
|
||
"error": readable,
|
||
"readable_error": readable,
|
||
"requested_input": str(requested_input),
|
||
"resolved_input": str(input_path),
|
||
"package_dir": str(package_dir),
|
||
"work_dir": str(work_dir),
|
||
"preflight_report": str(work_dir / "capture_preflight_report.json"),
|
||
}
|
||
(work_dir / "reconstruction_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
raise ValueError(readable)
|
||
_task_progress(store, task, 35, "准备重建输入", "正在写入输入清单、环境信息和命令参数")
|
||
input_report = {
|
||
"schema": "real2sim.reconstruction_input.v1",
|
||
"requested_input": str(requested_input),
|
||
"resolved_input": str(input_path),
|
||
"scan": scan_dataset(input_path),
|
||
"auto_prepare": bool(params.get("auto_prepare", False)),
|
||
"prepared_manifest": _read_optional_json(_prepared_manifest_path_for_input(input_path)),
|
||
}
|
||
(work_dir / "reconstruction_input.json").write_text(json.dumps(input_report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
env_report = {
|
||
"schema": "real2sim.reconstruction_environment.v1",
|
||
"python": sys.version,
|
||
"executable": sys.executable,
|
||
"ffmpeg": shutil.which("ffmpeg") or "",
|
||
"colmap": str(config.colmap_binary),
|
||
"gsplat_python": str(config.gsplat_python),
|
||
"anysplat_root": str(config.anysplat_root),
|
||
"anysplat_python": str(config.anysplat_python),
|
||
"runtime_root": str(config.runtime_root),
|
||
"runtime_env_script": str(config.runtime_env_script),
|
||
"cuda_home": str(config.cuda_home),
|
||
"gs_playground_root": str(config.gs_playground_root),
|
||
"time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
}
|
||
(work_dir / "environment.json").write_text(json.dumps(env_report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
reconstruction_python = config.anysplat_python if config.anysplat_python.exists() else config.gsplat_python if config.gsplat_python.exists() else Path(sys.executable)
|
||
quality_preset = _normalize_reconstruction_quality(str(params.get("quality_preset", "discoverse-high")))
|
||
command = [
|
||
str(reconstruction_python),
|
||
str(script),
|
||
"--output",
|
||
str(package_dir),
|
||
"--work-dir",
|
||
str(work_dir),
|
||
"--quality-preset",
|
||
quality_preset,
|
||
"--sfm-report",
|
||
"--capture-acceptance",
|
||
]
|
||
anysplat_root = Path(str(params.get("anysplat_root") or config.anysplat_root))
|
||
if str(anysplat_root):
|
||
command.extend(["--anysplat-root", str(anysplat_root)])
|
||
if str(config.anysplat_python):
|
||
command.extend(["--anysplat-python", str(config.anysplat_python)])
|
||
if Path(input_path).is_file() and Path(input_path).suffix.lower() in VIDEO_SUFFIXES:
|
||
command.extend(["--video", str(input_path)])
|
||
else:
|
||
command.extend(["--frames-dir", str(input_path)])
|
||
if params.get("pose_init_train", True):
|
||
command.append("--pose-init-train")
|
||
_append_discoverse_reconstruction_flags(command, params=params, quality_preset=quality_preset)
|
||
if params.get("allow_low_quality", False):
|
||
command.append("--allow-low-quality")
|
||
(work_dir / "reconstruction_command.json").write_text(json.dumps({"command": command, "cwd": str(config.gs_playground_root)}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
env = _env_with_paths(config)
|
||
_task_progress(store, task, 48, "运行真实重建", "AnySplat/上游重建命令正在执行")
|
||
try:
|
||
run_logged(command, cwd=config.gs_playground_root, log_path=run_dir / "logs" / "task.log", env=env, store=store, task_id=task["id"])
|
||
except Exception as exc:
|
||
log_tail = _tail_text(run_dir / "logs" / "task.log")
|
||
report = {
|
||
"schema": "real2sim.reconstruction_report.v1",
|
||
"status": "failed",
|
||
"error": str(exc),
|
||
"readable_error": _readable_failure_from_log(log_tail) or str(exc),
|
||
"log_tail": log_tail,
|
||
"requested_input": str(requested_input),
|
||
"resolved_input": str(input_path),
|
||
"package_dir": str(package_dir),
|
||
"work_dir": str(work_dir),
|
||
}
|
||
(work_dir / "reconstruction_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
raise
|
||
_task_progress(store, task, 82, "封装场景", "正在校验重建产物并生成统一场景包")
|
||
postprocess = _postprocess_discoverse_scene_assets(package_dir, work_dir, emit)
|
||
scene_path = _find_scene_json(package_dir) or (package_dir / "motrix_import.json")
|
||
manifest_path = _write_scene_manifest(package_dir, input_path=input_path, renderer=renderer, simulators=simulators, dry_run=False, external_assets=[])
|
||
report = {
|
||
"schema": "real2sim.reconstruction_report.v1",
|
||
"status": "succeeded",
|
||
"requested_input": str(requested_input),
|
||
"resolved_input": str(input_path),
|
||
"scene_path": str(scene_path),
|
||
"manifest_path": str(manifest_path),
|
||
"package_dir": str(package_dir),
|
||
"preflight_report": str(work_dir / "capture_preflight_report.json"),
|
||
"discoverse_postprocess": postprocess,
|
||
"asset_count": len(_candidate_asset_files(package_dir)),
|
||
}
|
||
(work_dir / "reconstruction_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
scene = store.create_scene(
|
||
name=params.get("scene_name", Path(input_path).stem),
|
||
path=scene_path,
|
||
metadata={"renderer": renderer, "simulators": simulators, "package": str(package_dir), "manifest_path": str(manifest_path)},
|
||
source_type="scene_reconstruction",
|
||
source_uri=str(input_path),
|
||
parent_task_id=task["id"],
|
||
manifest_path=manifest_path,
|
||
tags=[renderer, *simulators],
|
||
)
|
||
auto_cover: dict[str, Any] = {"status": "skipped"}
|
||
if bool(params.get("auto_cover", True)):
|
||
_task_progress(store, task, 90, "生成预览封面", "正在用 Scene Viewer 自动生成场景封面")
|
||
try:
|
||
auto_cover = _generate_scene_cover(config, store, task["id"], scene["id"], manifest_path, emit=emit)
|
||
except Exception as exc:
|
||
auto_cover = {"status": "warning", "error": str(exc)}
|
||
emit(f"auto scene cover skipped: {type(exc).__name__}: {exc}")
|
||
_index_artifacts(store, task["id"], run_dir)
|
||
_task_progress(store, task, 95, "索引产物", "重建报告、场景包和仿真产物已登记")
|
||
return {"scene_id": scene["id"], "scene_path": str(scene_path), "manifest_path": str(manifest_path), "package_dir": str(package_dir), "renderer": renderer, "simulators": simulators, "auto_cover": auto_cover}
|
||
|
||
|
||
def run_media_prepare(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
_task_progress(store, task, 18, "准备媒体", "正在解析视频、图片和数据清单")
|
||
prepared = prepare_media_dataset(config, store, params=task["params"], parent_task_id=task["id"], task_id=task["id"], run_dir=Path(task["run_dir"]), emit=emit)
|
||
_task_progress(store, task, 88, "登记数据集", "正在写入 prepared manifest 和平台索引")
|
||
_index_artifacts(store, task["id"], Path(prepared["dataset_path"]))
|
||
_task_progress(store, task, 96, "索引产物", "抽帧结果和数据清单已登记")
|
||
return prepared
|
||
|
||
|
||
def run_reconstruction_preflight(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
_task_progress(store, task, 20, "解析输入", "正在读取已准备帧数据")
|
||
input_path = _resolve_preflight_input(store, params)
|
||
_task_progress(store, task, 45, "质量预检", "正在检查帧数、分辨率、模糊度和视角变化")
|
||
report = build_reconstruction_preflight_report(config, store, params=params, input_path=input_path, run_dir=run_dir, emit=emit)
|
||
store.add_artifact(task_id=task["id"], name="capture_preflight_report.json", path=run_dir / "capture_preflight_report.json", kind="json")
|
||
_task_progress(store, task, 95, "生成报告", "预检报告已生成")
|
||
return {"ok": report["ok"], "report_path": str(run_dir / "capture_preflight_report.json"), "report": report}
|
||
|
||
|
||
def build_reconstruction_preflight_report(
|
||
config: PlatformConfig,
|
||
store: PlatformStore,
|
||
*,
|
||
params: dict[str, Any],
|
||
input_path: Path,
|
||
run_dir: Path,
|
||
emit: Callable[[str], None],
|
||
) -> dict[str, Any]:
|
||
run_dir.mkdir(parents=True, exist_ok=True)
|
||
min_frames = int(params.get("min_frames", RECONSTRUCTION_DEFAULT_MIN_FRAMES))
|
||
target_frame_count = int(params.get("target_frame_count", RECONSTRUCTION_DEFAULT_TARGET_FRAMES))
|
||
prepared_manifest = _read_optional_json(_prepared_manifest_path_for_input(input_path))
|
||
frame_paths = _frame_paths_for_preflight(input_path)
|
||
scan = scan_dataset(input_path) if input_path.exists() else {"path": str(input_path), "input_kind": "missing"}
|
||
runtime_checks = {
|
||
"gs_video_reconstruct": (config.gs_playground_root / "scripts/real2sim_video_motrix_scene.py").is_file(),
|
||
"anysplat_root": Path(str(params.get("anysplat_root") or config.anysplat_root)).is_dir(),
|
||
"anysplat_python": config.anysplat_python.exists(),
|
||
"colmap": shutil.which(str(config.colmap_binary)) is not None or config.colmap_binary.exists(),
|
||
"ffmpeg": shutil.which("ffmpeg") is not None,
|
||
}
|
||
quality = _frame_quality_summary(frame_paths)
|
||
failed_gates: list[str] = []
|
||
warnings: list[str] = []
|
||
if input_path.is_file() and input_path.suffix.lower() in VIDEO_SUFFIXES:
|
||
warnings.append("input is a video file; run media_prepare first for frame-level quality gates")
|
||
if len(frame_paths) < min_frames:
|
||
failed_gates.append("enough_frames")
|
||
warnings.append(f"only {len(frame_paths)} frame(s), expected at least {min_frames}; run media_prepare with more videos or choose a reconstruction capture")
|
||
if not runtime_checks["gs_video_reconstruct"]:
|
||
failed_gates.append("gs_video_reconstruct")
|
||
if not runtime_checks["anysplat_root"]:
|
||
failed_gates.append("anysplat_root")
|
||
if not runtime_checks["anysplat_python"]:
|
||
failed_gates.append("anysplat_python")
|
||
if not runtime_checks["ffmpeg"]:
|
||
failed_gates.append("ffmpeg")
|
||
if quality["width"] and quality["height"] and (quality["width"] < 320 or quality["height"] < 240):
|
||
warnings.append("input frames are below 320x240; reconstruction quality may be poor")
|
||
if quality["mean_frame_delta"] is not None and quality["mean_frame_delta"] < 2.0 and len(frame_paths) > 1:
|
||
warnings.append("adjacent frames are very similar; capture may lack camera motion/parallax")
|
||
|
||
allow_low_quality = bool(params.get("allow_low_quality", False))
|
||
ok = not failed_gates
|
||
readable_error = "" if ok else "reconstruction preflight failed: " + ", ".join(failed_gates)
|
||
readiness_score = _preflight_readiness_score(frame_count=len(frame_paths), min_frames=min_frames, runtime_checks=runtime_checks, warnings=warnings)
|
||
next_action = _preflight_next_action(ok=ok, allow_low_quality=allow_low_quality, failed_gates=failed_gates, warnings=warnings)
|
||
report = {
|
||
"schema": "real2sim.capture_preflight.v1",
|
||
"status": "succeeded" if ok else "failed",
|
||
"ok": ok,
|
||
"can_run": ok or allow_low_quality,
|
||
"allow_low_quality": allow_low_quality,
|
||
"readiness_score": readiness_score,
|
||
"next_action": next_action,
|
||
"readable_error": readable_error,
|
||
"input_path": str(input_path),
|
||
"scan": scan,
|
||
"prepared_manifest": prepared_manifest,
|
||
"frame_count": len(frame_paths),
|
||
"min_frames": min_frames,
|
||
"target_frame_count": target_frame_count,
|
||
"runtime_checks": runtime_checks,
|
||
"quality": quality,
|
||
"failed_gates": failed_gates,
|
||
"warnings": warnings,
|
||
"recommendations": _preflight_recommendations(failed_gates, warnings),
|
||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
}
|
||
(run_dir / "capture_preflight_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
emit(f"reconstruction preflight {'passed' if ok else 'failed'}: {len(frame_paths)} frame(s)")
|
||
return report
|
||
|
||
|
||
def run_hf_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
repo_id = str(params.get("repo_id", "")).strip()
|
||
repo_type = str(params.get("repo_type", "dataset")).strip() or "dataset"
|
||
if not repo_id or "/" not in repo_id:
|
||
raise ValueError("hf_import requires repo_id like namespace/name")
|
||
if repo_type != "dataset":
|
||
raise ValueError("hf_import only supports repo_type='dataset'")
|
||
revision = str(params.get("revision", "")).strip() or None
|
||
allow_patterns = _normalize_patterns(params.get("allow_patterns"))
|
||
dataset_id = uuid.uuid4().hex
|
||
local_name = _safe_name(str(params.get("local_name") or repo_id.split("/")[-1]))
|
||
target = config.datasets_dir / dataset_id
|
||
target.mkdir(parents=True, exist_ok=True)
|
||
emit(f"downloading {repo_id} to {target}\n")
|
||
_task_progress(store, task, 18, "下载数据", f"正在从 HuggingFace 下载 {repo_id}")
|
||
try:
|
||
from huggingface_hub import snapshot_download
|
||
except ImportError as exc:
|
||
raise RuntimeError("huggingface_hub is required for hf_import") from exc
|
||
try:
|
||
snapshot_download(
|
||
repo_id=repo_id,
|
||
repo_type=repo_type,
|
||
revision=revision,
|
||
allow_patterns=allow_patterns or None,
|
||
local_dir=str(target),
|
||
local_dir_use_symlinks=False,
|
||
)
|
||
except TypeError:
|
||
snapshot_download(repo_id=repo_id, repo_type=repo_type, revision=revision, allow_patterns=allow_patterns or None, local_dir=str(target))
|
||
scan = scan_dataset(target)
|
||
_task_progress(store, task, 78, "扫描数据", "正在统计图片、视频和元数据文件")
|
||
manifest = {
|
||
"schema": "real2sim.dataset_manifest.v1",
|
||
"id": dataset_id,
|
||
"name": local_name,
|
||
"source_type": "huggingface",
|
||
"source_uri": f"https://huggingface.co/datasets/{repo_id}",
|
||
"repo_id": repo_id,
|
||
"repo_type": repo_type,
|
||
"revision": revision or "main",
|
||
"allow_patterns": allow_patterns,
|
||
"scan": scan,
|
||
"files": _file_inventory(target, limit=200),
|
||
}
|
||
manifest_path = target / "dataset_manifest.json"
|
||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
dataset = store.create_dataset(
|
||
item_id=dataset_id,
|
||
name=local_name,
|
||
path=target,
|
||
scan=scan,
|
||
source_type="huggingface",
|
||
source_uri=manifest["source_uri"],
|
||
version=revision or "main",
|
||
parent_task_id=task["id"],
|
||
manifest_path=manifest_path,
|
||
tags=["huggingface", repo_id],
|
||
)
|
||
_index_artifacts(store, task["id"], target)
|
||
_task_progress(store, task, 95, "登记数据集", "数据集清单和平台索引已生成")
|
||
return {"dataset_id": dataset["id"], "dataset_path": str(target), "manifest_path": str(manifest_path), "scan": scan}
|
||
|
||
|
||
def run_bos_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
storage = BosStorage(config)
|
||
raw_prefix = normalize_relative_path(str(params.get("prefix") or ""))
|
||
prefix_parts = raw_prefix.split("/", 1)
|
||
area = prefix_parts[0] if prefix_parts[0] in ALLOWED_AREAS else "datasets"
|
||
relative_prefix = prefix_parts[1] if prefix_parts[0] in ALLOWED_AREAS and len(prefix_parts) > 1 else raw_prefix
|
||
_task_progress(store, task, 18, "读取 BOS", "正在列举授权前缀内的对象")
|
||
objects = storage.list_objects(area=area, relative_prefix=relative_prefix, max_keys=int(params.get("max_objects", 1000)))
|
||
objects = [item for item in objects if not item["key"].endswith("/")]
|
||
if not objects:
|
||
raise ValueError("BOS 前缀中没有可导入对象")
|
||
dataset_id = uuid.uuid4().hex
|
||
out_dir = config.datasets_dir / dataset_id
|
||
out_dir.mkdir(parents=True, exist_ok=False)
|
||
common_prefix = storage.object_key(area, relative_prefix)
|
||
_task_progress(store, task, 30, "下载数据", f"正在下载 {len(objects)} 个真实 BOS 对象")
|
||
for index, item in enumerate(objects, start=1):
|
||
relative = item["key"][len(common_prefix):].lstrip("/") or Path(item["key"]).name
|
||
storage.download_file(item["storage_uri"], out_dir / relative)
|
||
if index % max(1, len(objects) // 10) == 0:
|
||
_task_progress(store, task, 30 + int(45 * index / len(objects)), "下载数据", f"已下载 {index}/{len(objects)} 个对象")
|
||
scan = scan_dataset(out_dir)
|
||
digest = _directory_digest(out_dir)
|
||
source_uri = f"bos://{storage.bucket}/{common_prefix}"
|
||
manifest = {
|
||
"schema": "real2sim.dataset_manifest.v2",
|
||
"id": dataset_id,
|
||
"name": params.get("name") or Path(relative_prefix.rstrip("/")).name,
|
||
"source_type": "bos_prefix",
|
||
"storage_uri": source_uri,
|
||
"checksum": digest,
|
||
"version": digest[:12],
|
||
"object_count": len(objects),
|
||
"objects": objects,
|
||
"scan": scan,
|
||
}
|
||
manifest_path = out_dir / "dataset_manifest.json"
|
||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
dataset = store.create_dataset(
|
||
item_id=dataset_id,
|
||
name=str(manifest["name"]),
|
||
path=out_dir,
|
||
scan=scan,
|
||
source_type="bos_prefix",
|
||
source_uri=source_uri,
|
||
version=digest[:12],
|
||
parent_task_id=task["id"],
|
||
manifest_path=manifest_path,
|
||
metrics={"object_count": len(objects), "size_bytes": sum(int(item["size_bytes"]) for item in objects)},
|
||
tags=["bos", "imported"],
|
||
storage_uri=source_uri,
|
||
checksum=digest,
|
||
cache_state="local",
|
||
)
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "登记数据集", "BOS 数据及校验清单已登记")
|
||
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "storage_uri": source_uri, "object_count": len(objects), "checksum": digest}
|
||
|
||
|
||
def run_motrixsim_asset_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
"""Import the pinned MotrixSim package and prove both simulator runtimes load it."""
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
report_path = run_dir / "motrixsim_asset_import_report.json"
|
||
source = Path(str(params.get("source") or "/opt/motrixsim-docs"))
|
||
scene_id = hashlib.sha1(b"motrixsim:parlour_rm65_pick_place").hexdigest()[:32]
|
||
scripts = config.app_root / "scripts"
|
||
import_script = scripts / "import_motrixsim_assets.py"
|
||
runtime_script = scripts / "generate_motrixsim_runtime_scene.py"
|
||
preview_script = scripts / "generate_motrixsim_preview.py"
|
||
missing = [str(path) for path in (import_script, runtime_script, preview_script) if not path.is_file()]
|
||
if missing:
|
||
raise FileNotFoundError("MotrixSim asset pipeline scripts are missing: " + ", ".join(missing))
|
||
if not source.is_dir():
|
||
raise FileNotFoundError(f"pinned MotrixSim source checkout not found: {source}")
|
||
|
||
env = _env_with_paths(config)
|
||
log_path = run_dir / "logs" / "task.log"
|
||
_task_progress(store, task, 15, "导入官方资产", "正在复制固定版本的 MotrixSim 客厅、RM65 与 pick-place 资产")
|
||
run_logged(
|
||
[
|
||
sys.executable,
|
||
str(import_script),
|
||
"--source",
|
||
str(source),
|
||
"--name",
|
||
str(params.get("name") or "MotrixSim 客厅 RM65 Pick-Place"),
|
||
"--replace-index",
|
||
],
|
||
cwd=config.app_root,
|
||
log_path=log_path,
|
||
env=env,
|
||
store=store,
|
||
task_id=task["id"],
|
||
)
|
||
|
||
_task_progress(store, task, 48, "验证仿真运行时", "正在使用 MuJoCo 与 MotrixSim 真实加载并执行场景 step")
|
||
run_logged(
|
||
[sys.executable, str(runtime_script), "--scene-id", scene_id, "--steps", str(int(params.get("validation_steps", 120)))],
|
||
cwd=config.app_root,
|
||
log_path=log_path,
|
||
env=env,
|
||
store=store,
|
||
task_id=task["id"],
|
||
)
|
||
|
||
_task_progress(store, task, 72, "生成浏览器预览", "正在从官方网格资产生成可交互 GLB")
|
||
run_logged(
|
||
[sys.executable, str(preview_script), "--scene-id", scene_id, "--max-parlour-meshes", str(int(params.get("max_parlour_meshes", 240)))],
|
||
cwd=config.app_root,
|
||
log_path=log_path,
|
||
env=env,
|
||
store=store,
|
||
task_id=task["id"],
|
||
)
|
||
|
||
scene = store.get_scene(scene_id)
|
||
package_dir = Path(str((scene.get("metadata") or {}).get("package") or ""))
|
||
manifest_path = Path(str(scene.get("manifest_path") or ""))
|
||
if not package_dir.is_dir() or not manifest_path.is_file():
|
||
raise RuntimeError("MotrixSim asset pipeline completed without a valid scene package")
|
||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
runtime_validation = (scene.get("metadata") or {}).get("runtime_validation") or {}
|
||
if runtime_validation.get("mujoco") is not True or runtime_validation.get("motrixsim") is not True:
|
||
raise RuntimeError(f"MotrixSim scene did not pass both runtimes: {runtime_validation}")
|
||
primary = ((manifest.get("viewer_assets") or {}).get("primary") or {})
|
||
preview_path = Path(str(primary.get("path") or ""))
|
||
if not preview_path.is_file() or preview_path.stat().st_size == 0:
|
||
raise RuntimeError("MotrixSim GLB preview was not generated")
|
||
|
||
report = {
|
||
"schema": "real2sim.motrixsim_asset_import_report.v1",
|
||
"status": "succeeded",
|
||
"scene_id": scene_id,
|
||
"source": str(source),
|
||
"source_commit": _git_head(source),
|
||
"package_dir": str(package_dir),
|
||
"manifest_path": str(manifest_path),
|
||
"preview_path": str(preview_path),
|
||
"preview_size_bytes": preview_path.stat().st_size,
|
||
"runtime_validation": runtime_validation,
|
||
}
|
||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
store.add_artifact(task_id=task["id"], name=report_path.name, path=report_path, kind="json")
|
||
_index_artifacts(store, task["id"], package_dir)
|
||
_task_progress(store, task, 94, "登记场景资产", "官方资产、运行时报告和 GLB 预览已登记")
|
||
emit(f"MotrixSim asset package ready: scene_id={scene_id} preview={preview_path}")
|
||
return {
|
||
"scene_id": scene_id,
|
||
"scene_path": scene["path"],
|
||
"manifest_path": str(manifest_path),
|
||
"package_dir": str(package_dir),
|
||
"preview_path": str(preview_path),
|
||
"runtime_validation": runtime_validation,
|
||
}
|
||
|
||
|
||
def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
_task_progress(store, task, 20, "解析场景", "正在读取场景包和仿真产物")
|
||
scene_path = _scene_path(store, params)
|
||
if scene_path is None:
|
||
raise ValueError("simulator_validation requires scene_id or scene_path")
|
||
simulators = params.get("simulators", ["mujoco", "motrixsim"])
|
||
level = str(params.get("level") or "runtime_if_available")
|
||
if level not in SIM_VALIDATION_LEVELS:
|
||
raise ValueError("simulator_validation level must be smoke, runtime_if_available, or runtime_required")
|
||
root = scene_path.parent if scene_path.is_file() else scene_path
|
||
report = {
|
||
"schema": "real2sim.simulator_validation.v1",
|
||
"scene_path": str(scene_path),
|
||
"level": level,
|
||
"simulators": {},
|
||
"ok": True,
|
||
}
|
||
if "mujoco" in simulators:
|
||
_task_progress(store, task, 45, "校验 MuJoCo", "正在检查 MJCF 并尝试运行时加载")
|
||
candidates = [
|
||
root / "simulator_assets" / "rm65_table_pick_place.xml",
|
||
root / "scene_mujoco.xml",
|
||
root / "mujoco.xml",
|
||
root / "scene.xml",
|
||
root / "mjcf" / "scene.xml",
|
||
root / "mjcf" / "scene_mujoco.xml",
|
||
root / "mujoco" / "scene.xml",
|
||
*root.glob("*mujoco*.xml"),
|
||
*root.glob("mjcf/*.xml"),
|
||
*root.glob("mujoco/*.xml"),
|
||
*root.glob("simulator_assets/*.xml"),
|
||
]
|
||
report["simulators"]["mujoco"] = _validate_mujoco_asset(candidates, level=level)
|
||
if "motrixsim" in simulators:
|
||
_task_progress(store, task, 68, "校验 MotrixSim", "正在检查 MotrixSim 导入产物和运行时")
|
||
candidates = [
|
||
root / "motrix_scene.json",
|
||
root / "motrix_import.json",
|
||
root / "scene.json",
|
||
root / "simulator_assets" / "rm65_table_pick_place.xml",
|
||
root / "motrixsim" / "motrix_import.json",
|
||
root / "motrixsim" / "scene.json",
|
||
*root.glob("*motrix*.json"),
|
||
*root.glob("motrixsim/*.json"),
|
||
*root.glob("simulator_assets/*.xml"),
|
||
]
|
||
report["simulators"]["motrixsim"] = _validate_motrixsim_asset(candidates, level=level)
|
||
report["ok"] = all(item["ok"] for item in report["simulators"].values())
|
||
out_dir = run_dir / "validation"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
path = out_dir / "validation_report.json"
|
||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
store.add_artifact(task_id=task["id"], name="validation_report.json", path=path, kind="json")
|
||
_task_progress(store, task, 95, "生成报告", "仿真校验报告已生成")
|
||
return {"ok": report["ok"], "report_path": str(path), "report": report}
|
||
|
||
|
||
def run_expert_collection(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json"
|
||
out_dir = run_dir / "dataset"
|
||
policy_preset = str(params.get("policy_preset") or ("rm65_pick_place_6d_ik" if config.production_mode else "pick_and_place_v1"))
|
||
if policy_preset == "rm65_pick_place_6d_ik":
|
||
scene_path = _resolve_rm65_runtime_scene(store, params, scene_path)
|
||
required_successes = int(params.get("episodes", 50))
|
||
max_attempts = int(params.get("max_attempts", max(required_successes * 3, required_successes)))
|
||
command = [
|
||
sys.executable,
|
||
"-m",
|
||
"real2sim_platform.rm65_controller",
|
||
"--scene",
|
||
str(scene_path),
|
||
"--out",
|
||
str(out_dir),
|
||
"--required-successes",
|
||
str(required_successes),
|
||
"--max-attempts",
|
||
str(max_attempts),
|
||
"--steps",
|
||
str(int(params.get("steps", 200))),
|
||
"--seed",
|
||
str(int(params.get("seed", 7))),
|
||
"--width",
|
||
str(int(params.get("image_width", 320))),
|
||
"--height",
|
||
str(int(params.get("image_height", 240))),
|
||
]
|
||
_task_progress(store, task, 28, "物理采集", "RM65 正在通过 6D IK、双指接触和真实物理步进采集成功 episode")
|
||
run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src"]), store=store, task_id=task["id"])
|
||
manifest = _read_json(out_dir / "manifest.json")
|
||
dataset = store.create_dataset(
|
||
name=params.get("name", f"RM65 ACT dataset {task['id'][:8]}"),
|
||
path=out_dir,
|
||
scan={"input_kind": "discoverse_act_hdf5", "samples": manifest.get("successful_episodes"), "manifest": str(out_dir / "manifest.json")},
|
||
source_type="expert_collection",
|
||
source_uri=str(scene_path),
|
||
parent_task_id=task["id"],
|
||
manifest_path=out_dir / "manifest.json",
|
||
metrics={"samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate"), "policy_preset": policy_preset},
|
||
tags=["discoverse_act_hdf5", "rm65", "physical_contact", policy_preset],
|
||
)
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "登记数据集", "真实采集 HDF5、接触记录和成功报告已登记")
|
||
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate")}
|
||
if policy_preset != "pick_and_place_v1" or config.production_mode:
|
||
raise ValueError(f"unsupported expert policy preset: {policy_preset}")
|
||
_task_progress(store, task, 18, "准备采集", "正在加载场景、专家策略和机器人配置")
|
||
command = [
|
||
sys.executable,
|
||
"-m",
|
||
"real2sim_expert.collect",
|
||
"--scene",
|
||
str(scene_path),
|
||
"--out",
|
||
str(out_dir),
|
||
"--episodes",
|
||
str(int(params.get("episodes", 8))),
|
||
"--steps",
|
||
str(int(params.get("steps", 48))),
|
||
"--seed",
|
||
str(int(params.get("seed", 7))),
|
||
]
|
||
robot_config = params.get("robot_config") or str(config.expert_policy_root / "examples/single_arm_vla_robot.json")
|
||
task_config = params.get("task_config") or str(config.expert_policy_root / "examples/retail_pick_task.json")
|
||
command.extend(["--robot-config", robot_config, "--task-config", task_config])
|
||
_task_progress(store, task, 38, "专家采集", "专家策略正在生成 rollout 数据")
|
||
run_logged(command, cwd=config.expert_policy_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.expert_policy_root / "src"]), store=store, task_id=task["id"])
|
||
_task_progress(store, task, 82, "登记数据集", "正在读取采集 manifest 并生成训练数据集")
|
||
manifest = _read_json(out_dir / "manifest.json")
|
||
manifest["policy_preset"] = policy_preset
|
||
manifest["workers"] = 1
|
||
(out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
dataset = store.create_dataset(
|
||
name=params.get("name", f"expert dataset {task['id'][:8]}"),
|
||
path=out_dir,
|
||
scan={"input_kind": "expert_rollouts", "samples": manifest.get("samples"), "manifest": str(out_dir / "manifest.json")},
|
||
source_type="expert_collection",
|
||
source_uri=str(scene_path),
|
||
parent_task_id=task["id"],
|
||
manifest_path=out_dir / "manifest.json",
|
||
metrics={"samples": manifest.get("samples"), "policy_preset": policy_preset},
|
||
tags=["expert_rollouts", policy_preset],
|
||
)
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "索引产物", "采集数据和 manifest 已登记")
|
||
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "samples": manifest.get("samples")}
|
||
|
||
|
||
def run_training(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
_task_progress(store, task, 18, "准备训练", "正在加载训练数据集和训练配方")
|
||
dataset_path = _dataset_path(store, params)
|
||
out_dir = run_dir / "model"
|
||
recipe = str(params.get("recipe") or ("discoverse_act" if config.production_mode else "bc_v2"))
|
||
if recipe == "discoverse_act":
|
||
return _run_discoverse_act_training(config, store, task, dataset_path, out_dir)
|
||
recipe_config = {
|
||
"bc_v2": {"regularization": 1e-3, "base_model": "facebook/dinov2-small"},
|
||
"bc_fast": {"regularization": 1e-2, "base_model": "facebook/dinov2-small"},
|
||
}.get(recipe)
|
||
if recipe_config is None:
|
||
raise ValueError(f"unsupported training recipe: {recipe}")
|
||
command = [
|
||
sys.executable,
|
||
"-m",
|
||
"real2sim_train.train",
|
||
"--dataset",
|
||
str(dataset_path),
|
||
"--out",
|
||
str(out_dir),
|
||
"--epochs",
|
||
str(int(params.get("epochs", 100))),
|
||
"--regularization",
|
||
str(recipe_config["regularization"]),
|
||
"--base-model",
|
||
str(recipe_config["base_model"]),
|
||
"--split-seed",
|
||
str(int(params.get("split_seed", 17))),
|
||
]
|
||
_task_progress(store, task, 36, "模型训练", "训练进程正在运行")
|
||
run_logged(command, cwd=config.training_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.training_root / "src"]), store=store, task_id=task["id"])
|
||
_task_progress(store, task, 82, "生成模型卡", "正在读取训练指标和模型产物")
|
||
model_card = _read_json(out_dir / "model_card.json")
|
||
model_card["training"] = {
|
||
"recipe": recipe,
|
||
"epochs_requested": int(params.get("epochs", 100)),
|
||
"regularization": recipe_config["regularization"],
|
||
"base_model": recipe_config["base_model"],
|
||
}
|
||
(out_dir / "model_card.json").write_text(json.dumps(model_card, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
model = store.create_model(
|
||
name=params.get("name", f"policy {task['id'][:8]}"),
|
||
path=out_dir,
|
||
metadata=model_card,
|
||
source_type="training",
|
||
source_uri=str(dataset_path),
|
||
parent_task_id=task["id"],
|
||
manifest_path=out_dir / "model_card.json",
|
||
metrics=model_card.get("metrics", {}),
|
||
tags=["policy", recipe],
|
||
)
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "索引产物", "模型卡和训练产物已登记")
|
||
return {"model_id": model["id"], "model_path": str(out_dir), "metrics": model_card.get("metrics", {})}
|
||
|
||
|
||
def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
_task_progress(store, task, 18, "准备推理", "正在加载模型和场景")
|
||
scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json"
|
||
model_path = _model_path(store, params)
|
||
out_dir = run_dir / "inference"
|
||
model = store.get_model(str(params["model_id"])) if params.get("model_id") else None
|
||
model_recipe = str(((model or {}).get("metadata") or {}).get("training", {}).get("recipe") or params.get("recipe") or "")
|
||
if model_recipe == "discoverse_act":
|
||
scene_path = _resolve_rm65_runtime_scene(store, params, scene_path)
|
||
act_root = config.discoverse_root / "policies" / "act"
|
||
if _git_head(act_root) != config.act_commit:
|
||
raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}")
|
||
command = [
|
||
sys.executable,
|
||
"-m",
|
||
"real2sim_platform.act_inference",
|
||
"--scene",
|
||
str(scene_path),
|
||
"--model",
|
||
str(model_path),
|
||
"--act-root",
|
||
str(act_root),
|
||
"--out",
|
||
str(out_dir),
|
||
"--episodes",
|
||
str(int(params.get("episodes", 10))),
|
||
"--steps",
|
||
str(int(params.get("steps", 200))),
|
||
"--seed",
|
||
str(int(params.get("seed", 11))),
|
||
"--minimum-success-rate",
|
||
str(float(params.get("minimum_success_rate", 0.5))),
|
||
]
|
||
_task_progress(store, task, 38, "ACT 在线推理", "DISCOVERSE ACT 正在真实 MuJoCo 场景中闭环控制 RM65")
|
||
run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src", act_root, config.discoverse_root]), store=store, task_id=task["id"])
|
||
report = _read_json(out_dir / "inference_report.json")
|
||
if not report.get("actions_nonconstant"):
|
||
raise RuntimeError("ACT 推理动作是常量,拒绝登记为有效推理")
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "索引推理产物", "真实 rollout 视频、轨迹和成功率报告已登记")
|
||
return {"episodes": len(report.get("episodes", [])), "success_rate": report.get("success_rate"), "report": str(out_dir / "inference_report.json")}
|
||
if config.production_mode:
|
||
raise ValueError("生产模式只允许 DISCOVERSE ACT 模型在线推理")
|
||
command = [
|
||
sys.executable,
|
||
"-m",
|
||
"real2sim_infer.run",
|
||
"--scene",
|
||
str(scene_path),
|
||
"--model",
|
||
str(model_path),
|
||
"--out",
|
||
str(out_dir),
|
||
"--steps",
|
||
str(int(params.get("steps", 80))),
|
||
]
|
||
_task_progress(store, task, 42, "在线推理", "模型正在生成 rollout 轨迹")
|
||
run_logged(command, cwd=config.inference_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.inference_root / "src"]), store=store, task_id=task["id"])
|
||
_task_progress(store, task, 86, "生成轨迹", "正在读取推理结果和 viewer")
|
||
output = _read_json(out_dir / "prediction_rollout.json")
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "索引产物", "推理轨迹和 viewer 已登记")
|
||
return {"rollout_steps": len(output.get("rollout", [])), "viewer": str(out_dir / "viewer.html")}
|
||
|
||
|
||
def _run_discoverse_act_training(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], dataset_path: Path, out_dir: Path) -> dict[str, Any]:
|
||
params = task["params"]
|
||
run_dir = Path(task["run_dir"])
|
||
act_root = config.discoverse_root / "policies" / "act"
|
||
if _git_head(act_root) != config.act_commit:
|
||
raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}")
|
||
dataset_info = _inspect_act_dataset(dataset_path)
|
||
if dataset_info["episode_count"] < 2:
|
||
raise ValueError("DISCOVERSE ACT 至少需要 2 个真实 HDF5 episode")
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
config_root = run_dir / "generated_config"
|
||
task_name = f"real2sim_{task['id'][:12]}"
|
||
config_path = config_root / "configurations" / "task_configs" / f"{task_name}.py"
|
||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||
epochs = int(params.get("epochs", 100))
|
||
validate_every = max(1, min(int(params.get("validate_every", max(1, epochs // 5))), epochs))
|
||
save_every = max(1, min(int(params.get("save_every", validate_every)), epochs))
|
||
batch_size = max(1, min(int(params.get("batch_size", 8)), dataset_info["episode_count"] - 1))
|
||
chunk_size = max(2, min(int(params.get("chunk_size", 32)), dataset_info["episode_length"]))
|
||
core_dir = out_dir / "core"
|
||
checkpoint_dir = out_dir / "checkpoints"
|
||
config_path.write_text(
|
||
_act_task_config_source(
|
||
task_name=task_name,
|
||
dataset_path=dataset_path,
|
||
checkpoint_dir=checkpoint_dir,
|
||
stats_path=core_dir / "dataset_stats.pkl",
|
||
camera_names=dataset_info["camera_names"],
|
||
state_dim=dataset_info["state_dim"],
|
||
action_dim=dataset_info["action_dim"],
|
||
epochs=epochs,
|
||
batch_size=batch_size,
|
||
chunk_size=chunk_size,
|
||
validate_every=validate_every,
|
||
save_every=save_every,
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
command = [sys.executable, "policy_train.py", "-tn", task_name, "-ne", str(epochs), "-ve", str(validate_every), "-se", str(save_every), "--seed", str(int(params.get("seed", 1)))]
|
||
env = _env_with_paths(config, [config_root, act_root, config.discoverse_root])
|
||
env["MPLBACKEND"] = "Agg"
|
||
_task_progress(store, task, 34, "ACT 训练", f"正在使用 {dataset_info['episode_count']} 个真实 episode 训练 DISCOVERSE ACT")
|
||
run_logged(command, cwd=act_root, log_path=run_dir / "logs" / "task.log", env=env, store=store, task_id=task["id"])
|
||
checkpoint = core_dir / "policy_best.ckpt"
|
||
stats_path = core_dir / "dataset_stats.pkl"
|
||
if not checkpoint.is_file() or checkpoint.stat().st_size == 0 or not stats_path.is_file():
|
||
raise RuntimeError("DISCOVERSE ACT 训练结束但未生成非空 policy_best.ckpt 或 dataset_stats.pkl")
|
||
description = (core_dir / "description.txt").read_text(encoding="utf-8").strip() if (core_dir / "description.txt").is_file() else ""
|
||
best_loss = _parse_best_loss(description)
|
||
if best_loss is None or not math.isfinite(best_loss):
|
||
raise RuntimeError("ACT 训练指标不是有限数值")
|
||
model_card = {
|
||
"schema": "real2sim.model_card.v2",
|
||
"name": params.get("name", f"RM65 ACT {task['id'][:8]}"),
|
||
"framework": "DISCOVERSE ACT",
|
||
"training": {
|
||
"recipe": "discoverse_act",
|
||
"discoverse_commit": config.discoverse_commit,
|
||
"act_commit": config.act_commit,
|
||
"dataset_path": str(dataset_path),
|
||
"epochs": epochs,
|
||
"batch_size": batch_size,
|
||
"chunk_size": chunk_size,
|
||
"camera_names": dataset_info["camera_names"],
|
||
"state_dim": dataset_info["state_dim"],
|
||
"action_dim": dataset_info["action_dim"],
|
||
"image_color_order": "RGB (OpenCV-decoded BGR is converted by the generated ACT augmentor)",
|
||
"action_alignment": dataset_info["action_alignment"],
|
||
"control_hold_steps": dataset_info["control_hold_steps"],
|
||
"config_path": str(config_path),
|
||
"runtime_compatibility": "HDF5 ACT training; AIRBOT MCAP/hardware codecs remain optional and fail explicitly when requested",
|
||
},
|
||
"metrics": {"best_validation_loss": best_loss},
|
||
"artifacts": {"checkpoint": str(checkpoint), "dataset_stats": str(stats_path)},
|
||
"description": description,
|
||
}
|
||
card_path = out_dir / "model_card.json"
|
||
card_path.write_text(json.dumps(model_card, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
shutil.copy2(config_path, out_dir / "task_config.py")
|
||
model = store.create_model(
|
||
name=str(model_card["name"]),
|
||
path=out_dir,
|
||
metadata=model_card,
|
||
source_type="training",
|
||
source_uri=str(dataset_path),
|
||
parent_task_id=task["id"],
|
||
manifest_path=card_path,
|
||
metrics=model_card["metrics"],
|
||
tags=["policy", "discoverse_act", "rm65"],
|
||
)
|
||
_index_artifacts(store, task["id"], out_dir)
|
||
_task_progress(store, task, 95, "索引 ACT 模型", "checkpoint、损失曲线、模型卡和数据血缘已登记")
|
||
return {"model_id": model["id"], "model_path": str(out_dir), "metrics": model_card["metrics"], "checkpoint": str(checkpoint)}
|
||
|
||
|
||
def run_renderer_benchmark(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
run_dir = Path(task["run_dir"])
|
||
run_dir.mkdir(parents=True, exist_ok=True)
|
||
_task_progress(store, task, 45, "检测渲染器", "正在检查 Spark、Three.js 和 Aholo 可用性")
|
||
decision = renderer_decision()
|
||
path = run_dir / "renderer_decision.json"
|
||
path.write_text(json.dumps(decision, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
store.add_artifact(task_id=task["id"], name="renderer_decision.json", path=path, kind="json")
|
||
_task_progress(store, task, 95, "生成报告", "渲染器对比报告已生成")
|
||
return decision
|
||
|
||
|
||
def run_scene_cover_render(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
scene_id = str(params.get("scene_id") or "").strip()
|
||
if not scene_id:
|
||
raise ValueError("scene_cover_render requires scene_id")
|
||
scene = store.get_scene(scene_id)
|
||
manifest_path = _scene_manifest_path_for_scene(scene)
|
||
manifest = _read_optional_json(manifest_path)
|
||
if not manifest:
|
||
raise ValueError(f"scene manifest not found for scene {scene_id}")
|
||
_task_progress(store, task, 20, "准备封面", "正在解析场景清单和预览资源")
|
||
report = _generate_scene_cover(config, store, task["id"], scene_id, manifest_path, emit=emit)
|
||
_task_progress(store, task, 92, "封面完成", "场景封面已生成并写入清单")
|
||
return report
|
||
|
||
|
||
def run_scene_splat_optimize(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
params = task["params"]
|
||
scene_id = str(params.get("scene_id") or "").strip()
|
||
if not scene_id:
|
||
raise ValueError("scene_splat_optimize requires scene_id")
|
||
scene = store.get_scene(scene_id)
|
||
manifest_path = _scene_manifest_path_for_scene(scene)
|
||
manifest = _read_optional_json(manifest_path)
|
||
if not manifest:
|
||
raise ValueError(f"scene manifest not found for scene {scene_id}")
|
||
_task_progress(store, task, 18, "解析3DGS", "正在查找完整 3DGS 资产")
|
||
full_asset = _find_full_3dgs_manifest_asset(manifest)
|
||
if not full_asset:
|
||
raise ValueError("scene has no full 3DGS scene.ply asset to optimize")
|
||
run_dir = Path(task["run_dir"])
|
||
run_dir.mkdir(parents=True, exist_ok=True)
|
||
report_path = run_dir / "splat_optimization_report.json"
|
||
chrome = _find_chromium_binary()
|
||
_task_progress(store, task, 36, "检测打包能力", "正在检查 Spark/WebGL 解析能力和可落盘打包能力")
|
||
report = {
|
||
"schema": "real2sim.splat_optimization_report.v1",
|
||
"scene_id": scene_id,
|
||
"source_asset": full_asset,
|
||
"source_size_bytes": int(full_asset.get("size_bytes") or 0),
|
||
"source_size_label": full_asset.get("size_label") or "",
|
||
"viewer_probe": {},
|
||
"packed_output": None,
|
||
"pack_status": "not_available",
|
||
"recommendation": "The full 3DGS PLY is being converted to a browser-native SPZ asset.",
|
||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
}
|
||
_task_progress(store, task, 46, "导出SPZ", "正在把完整 3DGS PLY 转换成轻量 SPZ")
|
||
export = _export_scene_spz(config, store, task["id"], manifest_path, full_asset, emit)
|
||
report["spz_export"] = export
|
||
if export.get("ok"):
|
||
report["packed_output"] = export.get("asset")
|
||
report["pack_status"] = "spz_exported"
|
||
report["recommendation"] = "SPZ export succeeded. The scene manifest now includes a lightweight SPZ asset that should be preferred over the raw 3DGS PLY for browser preview."
|
||
if chrome and _scene_viewer_http_ready(config):
|
||
_task_progress(store, task, 55, "验证预览资产", "正在用 headless Scene Viewer 真实加载优化后的 SPZ/完整 3DGS")
|
||
report["viewer_probe"] = _probe_full_3dgs_with_chromium(config, chrome, scene_id, emit)
|
||
else:
|
||
report["viewer_probe"] = {"status": "skipped", "reason": "Chromium/Chrome binary or Scene Viewer HTTP endpoint not available"}
|
||
if report["pack_status"] == "spz_exported":
|
||
pass
|
||
elif report["viewer_probe"].get("loaded"):
|
||
report["pack_status"] = "viewer_load_succeeded_pack_export_unavailable"
|
||
report["recommendation"] = "Full 3DGS is valid and loaded by Spark. Keep it as manual full-quality asset until a server-side SPZ/KSPLAT exporter is installed."
|
||
else:
|
||
report["pack_status"] = "headless_viewer_probe_failed_pack_export_unavailable"
|
||
report["recommendation"] = "Server-side headless viewer could not reliably load this 118MB 3DGS within the probe budget. Keep pose-init preview as default and use manual full 3DGS loading until a native SPZ/KSPLAT exporter is installed."
|
||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
store.add_artifact(task_id=task["id"], name="splat_optimization_report.json", path=report_path, kind="json")
|
||
_task_progress(store, task, 92, "生成报告", "3DGS 优化能力报告已生成")
|
||
return report
|
||
|
||
|
||
def _export_scene_spz(config: PlatformConfig, store: PlatformStore, task_id: str, manifest_path: Path, source_asset: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||
source_path = Path(str(source_asset.get("path") or ""))
|
||
if not source_path.is_file():
|
||
return {"ok": False, "error": f"source 3DGS PLY not found: {source_path}"}
|
||
node = shutil.which("node")
|
||
if not node:
|
||
for candidate in [Path("/opt/images23dgs_app/node/bin/node"), Path("/usr/local/bin/node"), Path("/usr/bin/node")]:
|
||
if candidate.is_file():
|
||
node = str(candidate)
|
||
break
|
||
if not node:
|
||
return {"ok": False, "error": "node executable not found; cannot run Spark SPZ exporter"}
|
||
package_dir = manifest_path.parent
|
||
out_dir = package_dir / "optimized"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
output_path = out_dir / "scene.spz"
|
||
export_report_path = out_dir / "spz_export_report.json"
|
||
script = config.app_root / "scripts" / "convert_3dgs_ply_to_spz.mjs"
|
||
command = [
|
||
node,
|
||
str(script),
|
||
"--input",
|
||
str(source_path),
|
||
"--output",
|
||
str(output_path),
|
||
"--vendor-root",
|
||
str(config.app_root / "src" / "real2sim_platform" / "static" / "vendor"),
|
||
"--report",
|
||
str(export_report_path),
|
||
]
|
||
try:
|
||
completed = subprocess.run(command, cwd=str(config.app_root), text=True, capture_output=True, timeout=1800)
|
||
except Exception as exc:
|
||
return {"ok": False, "error": f"SPZ exporter failed to run: {type(exc).__name__}: {exc}"}
|
||
if completed.returncode != 0 or not output_path.is_file():
|
||
return {"ok": False, "error": (completed.stderr or completed.stdout or f"SPZ exporter exited {completed.returncode}")[-2000:]}
|
||
export_report = _read_optional_json(export_report_path)
|
||
asset = _viewer_asset(output_path, url=_run_static_url(output_path), role="optimized")
|
||
asset["optimized_from"] = str(source_path)
|
||
asset["autoload"] = output_path.stat().st_size <= VIEWER_AUTOLOAD_MAX_BYTES
|
||
asset["viewer_hint"] = "autoload" if asset["autoload"] else f"manual_load_large_asset_over_{VIEWER_AUTOLOAD_MAX_BYTES}_bytes"
|
||
manifest = _read_optional_json(manifest_path)
|
||
assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and item.get("path") == str(output_path))]
|
||
manifest["assets"] = [asset] + assets
|
||
manifest["optimized_assets"] = {"spz": asset, "source": str(source_path), "report": str(export_report_path)}
|
||
manifest["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||
manifest_path.write_text(json.dumps(enrich_scene_manifest_assets(manifest), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
store.add_artifact(task_id=task_id, name="scene.spz", path=output_path, kind="spz")
|
||
store.add_artifact(task_id=task_id, name="spz_export_report.json", path=export_report_path, kind="json")
|
||
emit(f"SPZ exported: {output_path.name} ({asset['size_label']})")
|
||
return {"ok": True, "asset": asset, "report": export_report}
|
||
|
||
|
||
def _find_full_3dgs_manifest_asset(manifest: dict[str, Any]) -> dict[str, Any] | None:
|
||
assets = [asset for asset in manifest.get("assets", []) if isinstance(asset, dict)]
|
||
candidates = []
|
||
for asset in assets:
|
||
name = str(asset.get("name") or "").lower()
|
||
path = str(asset.get("path") or asset.get("url") or "").lower()
|
||
size = int(asset.get("size_bytes") or 0)
|
||
if name == "scene.ply" and ("/3dgs/" in path or size > 50 * 1024 * 1024):
|
||
candidates.append(asset)
|
||
candidates.sort(key=lambda item: int(item.get("size_bytes") or 0), reverse=True)
|
||
return dict(candidates[0]) if candidates else None
|
||
|
||
|
||
def _probe_full_3dgs_with_chromium(config: PlatformConfig, chrome: Path, scene_id: str, emit: Callable[[str], None]) -> dict[str, Any]:
|
||
command = [
|
||
str(chrome),
|
||
"--headless=new",
|
||
"--disable-gpu",
|
||
"--no-sandbox",
|
||
"--window-size=960,640",
|
||
"--virtual-time-budget=110000",
|
||
"--dump-dom",
|
||
f"http://127.0.0.1:{config.port}/static/scene-viewer.html?scene_id={scene_id}&probe_full=1",
|
||
]
|
||
emit("splat/SPZ probe command: " + " ".join(command[:6]) + " ...")
|
||
completed = subprocess.run(command, text=True, capture_output=True, timeout=125)
|
||
text = (completed.stdout or "") + "\n" + (completed.stderr or "")
|
||
start = text.find('<pre id="probeResult"')
|
||
if start >= 0:
|
||
json_start = text.find("{", start)
|
||
json_end = text.find("</pre>", json_start)
|
||
raw = text[json_start:json_end if json_end >= 0 else None].strip()
|
||
try:
|
||
raw = raw.replace(""", '"').replace("&", "&").replace("<", "<").replace(">", ">")
|
||
return json.loads(raw)
|
||
except Exception:
|
||
emit("splat probe parse failed: " + raw[:300])
|
||
return {"loaded": False, "error": "probe did not return structured result", "stdout_tail": text[-800:]}
|
||
|
||
|
||
def _generate_scene_cover(config: PlatformConfig, store: PlatformStore, task_id: str, scene_id: str, manifest_path: Path, *, emit: Callable[[str], None]) -> dict[str, Any]:
|
||
manifest = _read_optional_json(manifest_path)
|
||
if not manifest:
|
||
raise ValueError(f"scene manifest not found for scene {scene_id}")
|
||
package_dir = manifest_path.parent
|
||
artifacts_dir = package_dir / "artifacts"
|
||
artifacts_dir.mkdir(parents=True, exist_ok=True)
|
||
cover_path = artifacts_dir / "scene_cover.png"
|
||
report_path = artifacts_dir / "scene_cover_report.json"
|
||
|
||
method = "chromium_scene_viewer"
|
||
warning = ""
|
||
chrome = _find_chromium_binary()
|
||
if chrome and _scene_viewer_http_ready(config):
|
||
candidates = _render_scene_cover_candidates(config, chrome, scene_id, artifacts_dir, emit)
|
||
if candidates:
|
||
best = max(candidates, key=lambda item: float(item.get("score") or 0))
|
||
shutil.copy2(best["path"], cover_path)
|
||
method = "chromium_scene_viewer_multiview"
|
||
else:
|
||
warning = "all Chromium cover candidates failed"
|
||
emit("chromium cover failed: " + warning)
|
||
method = "fallback_existing_preview"
|
||
_copy_existing_scene_preview(manifest, cover_path)
|
||
else:
|
||
warning = "Chromium/Chrome binary or Scene Viewer HTTP endpoint not available on this host; reused existing scene preview artifact."
|
||
method = "fallback_existing_preview"
|
||
_copy_existing_scene_preview(manifest, cover_path)
|
||
|
||
if not cover_path.is_file():
|
||
raise RuntimeError("scene cover render failed and no existing preview image was available")
|
||
cover_asset = _viewer_asset(cover_path, url=_run_static_url(cover_path), role="cover")
|
||
cover_asset["cover"] = True
|
||
assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and (item.get("cover") or item.get("name") == cover_asset["name"]))]
|
||
manifest["assets"] = [cover_asset] + assets
|
||
manifest["cover_asset"] = cover_asset
|
||
manifest["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||
manifest_path.write_text(json.dumps(enrich_scene_manifest_assets(manifest), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
report = {
|
||
"schema": "real2sim.scene_cover_report.v1",
|
||
"scene_id": scene_id,
|
||
"method": method,
|
||
"cover_path": str(cover_path),
|
||
"cover_url": _run_static_url(cover_path),
|
||
"candidates": candidates if chrome else [],
|
||
"warning": warning,
|
||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
}
|
||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
store.add_artifact(task_id=task_id, name="scene_cover.png", path=cover_path, kind="image")
|
||
for candidate in report.get("candidates", []):
|
||
candidate_path = Path(str(candidate.get("path") or ""))
|
||
if candidate_path.is_file():
|
||
store.add_artifact(task_id=task_id, name=candidate_path.name, path=candidate_path, kind="image")
|
||
store.add_artifact(task_id=task_id, name="scene_cover_report.json", path=report_path, kind="json")
|
||
return report
|
||
|
||
|
||
def _scene_viewer_http_ready(config: PlatformConfig) -> bool:
|
||
url = f"http://127.0.0.1:{config.port}/static/scene-viewer.html"
|
||
try:
|
||
with urlopen(url, timeout=0.75) as response:
|
||
return int(getattr(response, "status", 200)) < 500
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def renderer_decision() -> dict[str, Any]:
|
||
return {
|
||
"default": "spark",
|
||
"fallback": "aholo",
|
||
"reason": "Spark integrates directly with Three.js and mixed mesh/splat scenes, which fits simulator overlays. The platform falls back to a plain Three.js preview when the Spark bundle or input format is unavailable. Aholo is kept for huge 3DGS / chunked streaming LOD after assets are converted to its supported packed formats.",
|
||
"spark": {
|
||
"package": "@sparkjsdev/spark",
|
||
"bundle": str(SPARK_VENDOR_PATH),
|
||
"available": SPARK_VENDOR_PATH.is_file(),
|
||
"strengths": ["Three.js integration", "mixed splat and mesh rendering", "PLY/SPZ/SPLAT/KSPLAT/SOG support", "broad WebGL2 portability"],
|
||
},
|
||
"three_fallback": {
|
||
"bundle": str(THREE_VENDOR_PATH),
|
||
"available": THREE_VENDOR_PATH.is_file(),
|
||
"strengths": ["PLY point-cloud preview", "mesh preview", "small dependency surface"],
|
||
"constraint": "not a full 3DGS renderer",
|
||
},
|
||
"aholo": {
|
||
"package": "@manycore/aholo-viewer",
|
||
"strengths": ["high performance 3DGS", "mesh support", "chunked streaming LOD for large scenes"],
|
||
"constraint": "needs Aholo packing/LOD workflow for best results",
|
||
},
|
||
}
|
||
|
||
|
||
def _select_renderer(value: Any, package_dir: Path) -> str:
|
||
renderer = str(value or "spark").lower()
|
||
if renderer == "auto":
|
||
aholo_lod = package_dir / "aholo" / "manifest.json"
|
||
return "aholo" if aholo_lod.is_file() else "spark"
|
||
if renderer not in {"spark", "aholo"}:
|
||
raise ValueError(f"unsupported renderer: {renderer}")
|
||
return renderer
|
||
|
||
|
||
def run_logged(command: list[str], *, cwd: Path, log_path: Path, env: dict[str, str] | None = None, store: PlatformStore | None = None, task_id: str | None = None) -> None:
|
||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with log_path.open("a", encoding="utf-8") as log:
|
||
log.write("$ " + " ".join(command) + "\n")
|
||
log.flush()
|
||
process = subprocess.Popen(command, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env)
|
||
if store and task_id:
|
||
store.set_task_pid(task_id, process.pid)
|
||
store.update_task(task_id, message=f"子进程 {process.pid} 正在运行")
|
||
assert process.stdout is not None
|
||
while True:
|
||
line = process.stdout.readline()
|
||
if line:
|
||
log.write(line)
|
||
log.flush()
|
||
if store and task_id and store.get_task(task_id)["cancel_requested"] and process.poll() is None:
|
||
log.write("cancel requested; terminating subprocess\n")
|
||
log.flush()
|
||
process.terminate()
|
||
try:
|
||
process.wait(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
process.kill()
|
||
process.wait(timeout=5)
|
||
if store and task_id:
|
||
store.set_task_pid(task_id, None)
|
||
raise TaskCanceled("task canceled by user")
|
||
if process.poll() is not None:
|
||
break
|
||
for line in process.stdout:
|
||
log.write(line)
|
||
code = process.wait()
|
||
if store and task_id:
|
||
store.set_task_pid(task_id, None)
|
||
if code != 0:
|
||
raise RuntimeError(f"command failed with exit code {code}: {command[0]}")
|
||
|
||
|
||
def _resolve_preflight_input(store: PlatformStore, params: dict[str, Any]) -> Path:
|
||
input_path = _task_input_path(store, params)
|
||
dataset = store.get_dataset(str(params["dataset_id"])) if params.get("dataset_id") else None
|
||
prepared_manifest = _prepared_manifest_for_dataset(dataset) if dataset else _read_optional_json(_prepared_manifest_path_for_input(input_path))
|
||
if prepared_manifest:
|
||
frames_dir = Path(str(prepared_manifest.get("frames_dir") or input_path / "frames"))
|
||
if frames_dir.exists():
|
||
return frames_dir
|
||
return input_path
|
||
|
||
|
||
def _frame_paths_for_preflight(path: Path) -> list[Path]:
|
||
if path.is_file() and path.suffix.lower() in IMAGE_SUFFIXES:
|
||
return [path]
|
||
if not path.is_dir():
|
||
return []
|
||
return sorted(item for item in path.iterdir() if item.is_file() and item.suffix.lower() in IMAGE_SUFFIXES)
|
||
|
||
|
||
def _frame_quality_summary(frame_paths: list[Path], *, sample_limit: int = 12) -> dict[str, Any]:
|
||
summary: dict[str, Any] = {
|
||
"width": None,
|
||
"height": None,
|
||
"sampled_frames": 0,
|
||
"mean_luma": None,
|
||
"mean_blur_score": None,
|
||
"mean_frame_delta": None,
|
||
}
|
||
if not frame_paths:
|
||
return summary
|
||
try:
|
||
from PIL import Image, ImageChops, ImageFilter, ImageStat
|
||
except ImportError:
|
||
return {**summary, "warning": "Pillow is required for frame quality metrics"}
|
||
if len(frame_paths) <= sample_limit:
|
||
samples = frame_paths
|
||
else:
|
||
step = max(1, math.floor(len(frame_paths) / sample_limit))
|
||
samples = frame_paths[::step][:sample_limit]
|
||
luma_values: list[float] = []
|
||
blur_values: list[float] = []
|
||
delta_values: list[float] = []
|
||
previous = None
|
||
for frame in samples:
|
||
try:
|
||
with Image.open(frame) as image:
|
||
rgb = image.convert("RGB")
|
||
if summary["width"] is None:
|
||
summary["width"], summary["height"] = rgb.size
|
||
gray = rgb.convert("L")
|
||
luma_values.append(float(ImageStat.Stat(gray).mean[0]))
|
||
edges = gray.filter(ImageFilter.FIND_EDGES)
|
||
blur_values.append(float(ImageStat.Stat(edges).var[0]))
|
||
small = gray.resize((64, 64))
|
||
if previous is not None:
|
||
diff = ImageChops.difference(previous, small)
|
||
delta_values.append(float(ImageStat.Stat(diff).mean[0]))
|
||
previous = small
|
||
except OSError:
|
||
continue
|
||
summary["sampled_frames"] = len(luma_values)
|
||
summary["mean_luma"] = round(sum(luma_values) / len(luma_values), 3) if luma_values else None
|
||
summary["mean_blur_score"] = round(sum(blur_values) / len(blur_values), 3) if blur_values else None
|
||
summary["mean_frame_delta"] = round(sum(delta_values) / len(delta_values), 3) if delta_values else None
|
||
return summary
|
||
|
||
|
||
def _preflight_recommendations(failed_gates: list[str], warnings: list[str]) -> list[str]:
|
||
recommendations: list[str] = []
|
||
if "enough_frames" in failed_gates:
|
||
recommendations.append("Run media_prepare with target_frame_count=80 and max_videos high enough to produce at least 30 reconstructable frames.")
|
||
recommendations.append("Use a reconstruction-oriented capture with camera motion/parallax; xarm policy videos may be too short or too static.")
|
||
if "anysplat_root" in failed_gates:
|
||
recommendations.append("Set anysplat_root or REAL2SIM_ANYSPLAT_ROOT to /opt/real2sim_paper_runtime/AnySplat on Wuying.")
|
||
if "anysplat_python" in failed_gates:
|
||
recommendations.append("Set anysplat_python to /opt/real2sim_paper_runtime/envs/anysplat/bin/python.")
|
||
if "gs_video_reconstruct" in failed_gates:
|
||
recommendations.append("Set gs_playground_root to the checkout containing scripts/real2sim_video_motrix_scene.py.")
|
||
if not recommendations and warnings:
|
||
recommendations.append("Review capture warnings before starting a long reconstruction job.")
|
||
return recommendations
|
||
|
||
|
||
def _preflight_readiness_score(*, frame_count: int, min_frames: int, runtime_checks: dict[str, bool], warnings: list[str]) -> int:
|
||
score = 100
|
||
if frame_count < min_frames:
|
||
score -= min(60, int((min_frames - frame_count) / max(min_frames, 1) * 60) + 20)
|
||
missing_runtime = [name for name, ok in runtime_checks.items() if not ok]
|
||
score -= 15 * len(missing_runtime)
|
||
score -= min(20, 5 * len(warnings))
|
||
return max(0, min(100, score))
|
||
|
||
|
||
def _preflight_next_action(*, ok: bool, allow_low_quality: bool, failed_gates: list[str], warnings: list[str]) -> str:
|
||
if ok:
|
||
return "ready_for_scene_reconstruction"
|
||
if "enough_frames" in failed_gates:
|
||
return "run_media_prepare_or_recapture_more_frames"
|
||
if any(name in failed_gates for name in ["anysplat_root", "anysplat_python", "gs_video_reconstruct"]):
|
||
return "fix_reconstruction_runtime_configuration"
|
||
if allow_low_quality:
|
||
return "can_run_with_allow_low_quality_but_expect_poor_reconstruction"
|
||
if warnings:
|
||
return "review_capture_warnings_before_reconstruction"
|
||
return "fix_failed_preflight_gates"
|
||
|
||
|
||
def _task_input_path(store: PlatformStore, params: dict[str, Any]) -> Path:
|
||
if params.get("input_path"):
|
||
return Path(str(params["input_path"]))
|
||
if params.get("dataset_id"):
|
||
return Path(store.get_dataset(str(params["dataset_id"]))["path"])
|
||
raise ValueError("scene_reconstruction requires input_path or dataset_id")
|
||
|
||
|
||
def _resolve_reconstruction_input(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> Path:
|
||
params = task["params"]
|
||
dataset = store.get_dataset(str(params["dataset_id"])) if params.get("dataset_id") else None
|
||
input_path = Path(dataset["path"]) if dataset else Path(str(params.get("input_path", "")))
|
||
prepared_manifest = _prepared_manifest_for_dataset(dataset) if dataset else _read_optional_json(_prepared_manifest_path_for_input(input_path))
|
||
if prepared_manifest:
|
||
frames_dir = Path(str(prepared_manifest.get("frames_dir") or input_path / "frames"))
|
||
if _has_reconstruction_images(frames_dir):
|
||
return frames_dir
|
||
raise ValueError(f"prepared dataset has no reconstructable frames: {frames_dir}")
|
||
if params.get("auto_prepare"):
|
||
emit("auto_prepare enabled; preparing media before reconstruction")
|
||
prepared = prepare_media_dataset(config, store, params=params, parent_task_id=task["id"], task_id=task["id"], run_dir=Path(task["run_dir"]) / "prepared_media", emit=emit)
|
||
return Path(prepared["frames_dir"])
|
||
if input_path.is_file() and input_path.suffix.lower() in VIDEO_SUFFIXES:
|
||
return input_path
|
||
if _has_reconstruction_images(input_path):
|
||
return input_path
|
||
scan = scan_dataset(input_path) if input_path.exists() else {"input_kind": "missing", "path": str(input_path)}
|
||
if scan.get("video_count") or scan.get("parquet_count") or scan.get("input_kind") in {"video", "parquet_video"}:
|
||
raise ValueError("scene_reconstruction non-dry input contains video/parquet data; run media_prepare first or set auto_prepare=true")
|
||
raise ValueError("scene_reconstruction non-dry requires a video file or a prepared/images frame directory; use dry_run=true for PLY/3DGS-only preview assets")
|
||
|
||
|
||
def prepare_media_dataset(
|
||
config: PlatformConfig,
|
||
store: PlatformStore,
|
||
*,
|
||
params: dict[str, Any],
|
||
parent_task_id: str,
|
||
task_id: str,
|
||
run_dir: Path,
|
||
emit: Callable[[str], None],
|
||
) -> dict[str, Any]:
|
||
source_dataset = store.get_dataset(str(params["dataset_id"])) if params.get("dataset_id") else None
|
||
source_path = Path(source_dataset["path"]) if source_dataset else Path(str(params.get("input_path", "")))
|
||
if not source_path.exists() and not (str(source_path).startswith("http://") or str(source_path).startswith("https://")):
|
||
raise FileNotFoundError(source_path)
|
||
sample_fps = float(params.get("sample_fps", MEDIA_PREPARE_DEFAULT_SAMPLE_FPS))
|
||
max_frames_per_video = int(params.get("max_frames_per_video", MEDIA_PREPARE_DEFAULT_MAX_FRAMES))
|
||
max_videos = int(params.get("max_videos", 1))
|
||
target_frame_count = int(params.get("target_frame_count", 0) or 0)
|
||
min_total_frames = int(params.get("min_total_frames", 0) or 0)
|
||
camera_label = str(params.get("camera_label") or MEDIA_PREPARE_DEFAULT_CAMERA)
|
||
source_column = str(params.get("source_column") or "").strip()
|
||
if sample_fps <= 0:
|
||
raise ValueError("sample_fps must be > 0")
|
||
if max_frames_per_video <= 0:
|
||
raise ValueError("max_frames_per_video must be > 0")
|
||
if max_videos <= 0:
|
||
raise ValueError("max_videos must be > 0")
|
||
if target_frame_count < 0:
|
||
raise ValueError("target_frame_count must be >= 0")
|
||
if min_total_frames < 0:
|
||
raise ValueError("min_total_frames must be >= 0")
|
||
|
||
dataset_id = uuid.uuid4().hex
|
||
output_name = _safe_name(str(params.get("output_name") or f"prepared-{source_path.stem or 'media'}"))
|
||
target = config.datasets_dir / dataset_id
|
||
frames_dir = target / "frames"
|
||
thumbnails_dir = target / "thumbnails"
|
||
target.mkdir(parents=True, exist_ok=True)
|
||
frames_dir.mkdir(parents=True, exist_ok=True)
|
||
thumbnails_dir.mkdir(parents=True, exist_ok=True)
|
||
run_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
source_manifest = _read_optional_json(source_path / "dataset_manifest.json") if source_path.is_dir() else {}
|
||
warnings: list[str] = []
|
||
effective_max_videos = max_videos
|
||
if target_frame_count:
|
||
effective_max_videos = max(max_videos, math.ceil(target_frame_count / max_frames_per_video))
|
||
video_sources = _collect_video_sources(source_path, source_manifest, camera_label=camera_label, source_column=source_column, limit=effective_max_videos)
|
||
copied_images = 0
|
||
video_frame_counts: list[dict[str, Any]] = []
|
||
if video_sources:
|
||
emit(f"media_prepare selected {len(video_sources)} video source(s)")
|
||
for index, source in enumerate(video_sources):
|
||
remaining = target_frame_count - copied_images if target_frame_count else max_frames_per_video
|
||
per_video_limit = max(1, min(max_frames_per_video, remaining)) if target_frame_count else max_frames_per_video
|
||
frame_count = _extract_video_frames(source, frames_dir, video_index=index, sample_fps=sample_fps, max_frames=per_video_limit, store=store, task_id=task_id, log_path=run_dir / "logs" / "task.log")
|
||
copied_images += frame_count
|
||
video_frame_counts.append({"source": str(source), "frame_count": frame_count})
|
||
if target_frame_count and copied_images >= target_frame_count:
|
||
break
|
||
else:
|
||
copied_images = _copy_image_frames(source_path, frames_dir, max_frames=target_frame_count or max_frames_per_video * max_videos)
|
||
if copied_images == 0:
|
||
warnings.append("no local videos or image frames found")
|
||
if copied_images == 0:
|
||
raise ValueError("media_prepare did not produce any frames; check camera_label/source_column or input media")
|
||
if min_total_frames and copied_images < min_total_frames:
|
||
raise ValueError(f"media_prepare produced {copied_images} frame(s), below min_total_frames={min_total_frames}")
|
||
|
||
width, height = _image_size(next(iter(sorted(frames_dir.glob("*"))), None))
|
||
_write_thumbnails(frames_dir, thumbnails_dir)
|
||
manifest = {
|
||
"schema": "real2sim.prepared_manifest.v1",
|
||
"id": dataset_id,
|
||
"name": output_name,
|
||
"source_dataset_id": source_dataset["id"] if source_dataset else "",
|
||
"source_uri": source_dataset["source_uri"] if source_dataset else str(source_path),
|
||
"source_path": str(source_path),
|
||
"video_urls": [str(source) for source in video_sources],
|
||
"video_frame_counts": video_frame_counts,
|
||
"frames_dir": str(frames_dir),
|
||
"frame_count": copied_images,
|
||
"target_frame_count": target_frame_count,
|
||
"min_total_frames": min_total_frames,
|
||
"camera_label": camera_label,
|
||
"source_column": source_column,
|
||
"sample_fps": sample_fps,
|
||
"fps": sample_fps,
|
||
"width": width,
|
||
"height": height,
|
||
"parent_task_id": parent_task_id,
|
||
"warnings": warnings,
|
||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
}
|
||
manifest_path = target / "prepared_manifest.json"
|
||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
(target / "source_manifest.json").write_text(json.dumps(source_manifest or {}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
scan = scan_dataset(target)
|
||
dataset = store.create_dataset(
|
||
item_id=dataset_id,
|
||
name=output_name,
|
||
path=target,
|
||
scan=scan,
|
||
source_type="prepared_media",
|
||
source_uri=manifest["source_uri"],
|
||
version=str(params.get("revision") or source_dataset["version"] if source_dataset else ""),
|
||
parent_task_id=parent_task_id,
|
||
manifest_path=manifest_path,
|
||
metrics={"frame_count": copied_images, "sample_fps": sample_fps, "video_count": len(video_sources)},
|
||
tags=["prepared_media", camera_label],
|
||
)
|
||
return {
|
||
"dataset_id": dataset["id"],
|
||
"dataset_path": str(target),
|
||
"manifest_path": str(manifest_path),
|
||
"frames_dir": str(frames_dir),
|
||
"frame_count": copied_images,
|
||
"video_urls": [str(source) for source in video_sources],
|
||
"video_frame_counts": video_frame_counts,
|
||
"warnings": warnings,
|
||
}
|
||
|
||
|
||
def _collect_video_sources(source_path: Path, source_manifest: dict[str, Any], *, camera_label: str, source_column: str, limit: int) -> list[str]:
|
||
sources: list[str] = []
|
||
if str(source_path).startswith(("http://", "https://")):
|
||
sources.append(str(source_path))
|
||
elif source_path.is_file() and source_path.suffix.lower() in VIDEO_SUFFIXES:
|
||
sources.append(str(source_path))
|
||
elif source_path.is_dir():
|
||
sources.extend(str(path) for path in sorted(source_path.rglob("*")) if path.is_file() and path.suffix.lower() in VIDEO_SUFFIXES)
|
||
manifest_urls = _extract_video_url_candidates(source_manifest)
|
||
sources.extend(manifest_urls)
|
||
sources.extend(_video_urls_from_parquet(source_path, camera_label=camera_label, source_column=source_column))
|
||
repo_id = str(source_manifest.get("repo_id") or "").strip()
|
||
revision = str(source_manifest.get("revision") or "main").strip() or "main"
|
||
if repo_id:
|
||
sources.extend(_video_urls_from_hf_files(repo_id, revision=revision, camera_label=camera_label, limit=limit))
|
||
sources.extend(_video_urls_from_hf_viewer(repo_id, camera_label=camera_label, limit=limit))
|
||
deduped: list[str] = []
|
||
for source in sources:
|
||
if camera_label and camera_label not in source and source.startswith(("http://", "https://")):
|
||
continue
|
||
if source not in deduped:
|
||
deduped.append(source)
|
||
if len(deduped) >= limit:
|
||
break
|
||
return deduped
|
||
|
||
|
||
def _video_urls_from_hf_files(repo_id: str, *, revision: str, camera_label: str, limit: int) -> list[str]:
|
||
try:
|
||
from huggingface_hub import HfApi
|
||
files = HfApi().list_repo_files(repo_id=repo_id, repo_type="dataset", revision=revision)
|
||
except Exception:
|
||
return []
|
||
urls: list[str] = []
|
||
for filename in sorted(files):
|
||
lower = filename.lower()
|
||
if not any(lower.endswith(suffix) for suffix in VIDEO_SUFFIXES):
|
||
continue
|
||
if camera_label and camera_label not in filename:
|
||
continue
|
||
urls.append(f"https://huggingface.co/datasets/{repo_id}/resolve/{revision}/{filename}")
|
||
if len(urls) >= limit:
|
||
break
|
||
return urls
|
||
|
||
|
||
def _video_urls_from_hf_viewer(repo_id: str, *, camera_label: str, limit: int) -> list[str]:
|
||
dataset = quote(repo_id, safe="")
|
||
try:
|
||
with urlopen(f"https://datasets-server.huggingface.co/splits?dataset={dataset}", timeout=20) as response:
|
||
splits_payload = json.loads(response.read().decode("utf-8"))
|
||
except Exception:
|
||
return []
|
||
urls: list[str] = []
|
||
for split in splits_payload.get("splits", []):
|
||
config = quote(str(split.get("config") or "default"), safe="")
|
||
split_name = quote(str(split.get("split") or "train"), safe="")
|
||
try:
|
||
with urlopen(f"https://datasets-server.huggingface.co/first-rows?dataset={dataset}&config={config}&split={split_name}", timeout=30) as response:
|
||
rows_payload = json.loads(response.read().decode("utf-8"))
|
||
except Exception:
|
||
continue
|
||
for row in rows_payload.get("rows", []):
|
||
for candidate in _extract_video_url_candidates(row.get("row", {})):
|
||
if camera_label and camera_label not in candidate:
|
||
continue
|
||
if candidate not in urls:
|
||
urls.append(candidate)
|
||
if len(urls) >= limit:
|
||
return urls
|
||
return urls
|
||
|
||
|
||
def _video_urls_from_parquet(source_path: Path, *, camera_label: str, source_column: str) -> list[str]:
|
||
parquet_files: list[Path] = []
|
||
if source_path.is_file() and source_path.suffix.lower() in PARQUET_SUFFIXES:
|
||
parquet_files = [source_path]
|
||
elif source_path.is_dir():
|
||
parquet_files = sorted(source_path.rglob("*.parquet"))
|
||
if not parquet_files:
|
||
return []
|
||
try:
|
||
import pyarrow.parquet as pq
|
||
except ImportError as exc:
|
||
raise RuntimeError("pyarrow is required to read HuggingFace parquet video manifests") from exc
|
||
urls: list[str] = []
|
||
for parquet in parquet_files:
|
||
table = pq.read_table(parquet)
|
||
for row in table.to_pylist():
|
||
values = [_nested_value(row, source_column)] if source_column else [row]
|
||
for value in values:
|
||
for candidate in _extract_video_url_candidates(value):
|
||
if camera_label and camera_label not in candidate:
|
||
continue
|
||
if candidate not in urls:
|
||
urls.append(candidate)
|
||
return urls
|
||
|
||
|
||
def _extract_video_url_candidates(value: Any) -> list[str]:
|
||
found: list[str] = []
|
||
if isinstance(value, str):
|
||
lower = value.lower().split("?", 1)[0]
|
||
if value.startswith(("http://", "https://")) and any(lower.endswith(suffix) for suffix in VIDEO_SUFFIXES):
|
||
found.append(value)
|
||
elif isinstance(value, dict):
|
||
if "src" in value:
|
||
found.extend(_extract_video_url_candidates(value["src"]))
|
||
for nested in value.values():
|
||
found.extend(_extract_video_url_candidates(nested))
|
||
elif isinstance(value, list):
|
||
for item in value:
|
||
found.extend(_extract_video_url_candidates(item))
|
||
return found
|
||
|
||
|
||
def _nested_value(row: dict[str, Any], source_column: str) -> Any:
|
||
value: Any = row
|
||
for part in source_column.split("."):
|
||
if isinstance(value, dict):
|
||
value = value.get(part)
|
||
else:
|
||
return None
|
||
return value
|
||
|
||
|
||
def _extract_video_frames(source: str, frames_dir: Path, *, video_index: int, sample_fps: float, max_frames: int, store: PlatformStore, task_id: str, log_path: Path) -> int:
|
||
ffmpeg = shutil.which("ffmpeg")
|
||
if not ffmpeg:
|
||
raise RuntimeError("ffmpeg is required for media_prepare video extraction")
|
||
ffmpeg_source = _materialize_video_source(source, frames_dir.parent / "sources", log_path)
|
||
pattern = frames_dir / f"video{video_index:02d}_%06d.jpg"
|
||
before = {path.name for path in frames_dir.glob("*.jpg")}
|
||
command = [
|
||
ffmpeg,
|
||
"-hide_banner",
|
||
"-y",
|
||
"-i",
|
||
ffmpeg_source,
|
||
"-vf",
|
||
f"fps={sample_fps}",
|
||
"-frames:v",
|
||
str(max_frames),
|
||
"-q:v",
|
||
"2",
|
||
str(pattern),
|
||
]
|
||
run_logged(command, cwd=frames_dir.parent, log_path=log_path, store=store, task_id=task_id)
|
||
after = {path.name for path in frames_dir.glob("*.jpg")}
|
||
return len(after - before)
|
||
|
||
|
||
def _materialize_video_source(source: str, source_dir: Path, log_path: Path) -> str:
|
||
if not source.startswith(("http://", "https://")):
|
||
return source
|
||
source_dir.mkdir(parents=True, exist_ok=True)
|
||
parsed = urlparse(source)
|
||
filename = _safe_name(Path(parsed.path).name or "video.mp4")
|
||
target = source_dir / filename
|
||
if target.is_file() and target.stat().st_size > 0:
|
||
return str(target)
|
||
_append_log(log_path, f"downloading remote video source to {target}")
|
||
hf_info = _parse_hf_resolve_url(parsed)
|
||
if hf_info:
|
||
try:
|
||
from huggingface_hub import hf_hub_download
|
||
downloaded = hf_hub_download(
|
||
repo_id=hf_info["repo_id"],
|
||
repo_type="dataset",
|
||
revision=hf_info["revision"],
|
||
filename=hf_info["filename"],
|
||
local_dir=str(source_dir),
|
||
local_dir_use_symlinks=False,
|
||
)
|
||
downloaded_path = Path(downloaded)
|
||
if downloaded_path != target and downloaded_path.is_file():
|
||
shutil.copy2(downloaded_path, target)
|
||
return str(target if target.is_file() else downloaded_path)
|
||
except Exception as exc:
|
||
_append_log(log_path, f"huggingface_hub download failed; falling back to urlopen: {exc}")
|
||
with urlopen(source, timeout=120) as response, target.open("wb") as handle:
|
||
shutil.copyfileobj(response, handle)
|
||
return str(target)
|
||
|
||
|
||
def _parse_hf_resolve_url(parsed) -> dict[str, str] | None:
|
||
if parsed.netloc != "huggingface.co":
|
||
return None
|
||
parts = [unquote(part) for part in parsed.path.strip("/").split("/")]
|
||
if len(parts) < 6 or parts[0] != "datasets" or parts[3] != "resolve":
|
||
return None
|
||
return {
|
||
"repo_id": f"{parts[1]}/{parts[2]}",
|
||
"revision": parts[4],
|
||
"filename": "/".join(parts[5:]),
|
||
}
|
||
|
||
|
||
def _copy_image_frames(source_path: Path, frames_dir: Path, *, max_frames: int) -> int:
|
||
if source_path.is_file() and source_path.suffix.lower() in IMAGE_SUFFIXES:
|
||
image_paths = [source_path]
|
||
elif source_path.is_dir():
|
||
image_paths = sorted(path for path in source_path.rglob("*") if path.is_file() and path.suffix.lower() in IMAGE_SUFFIXES)
|
||
else:
|
||
image_paths = []
|
||
count = 0
|
||
for count, image_path in enumerate(image_paths[:max_frames], start=1):
|
||
suffix = image_path.suffix.lower() if image_path.suffix.lower() in IMAGE_SUFFIXES else ".jpg"
|
||
shutil.copy2(image_path, frames_dir / f"frame_{count:06d}{suffix}")
|
||
return count
|
||
|
||
|
||
def _write_thumbnails(frames_dir: Path, thumbnails_dir: Path) -> None:
|
||
try:
|
||
from PIL import Image
|
||
except ImportError:
|
||
return
|
||
for index, frame in enumerate(sorted(frames_dir.glob("*"))[:6], start=1):
|
||
try:
|
||
with Image.open(frame) as image:
|
||
image.thumbnail((320, 180))
|
||
image.convert("RGB").save(thumbnails_dir / f"thumb_{index:02d}.jpg", quality=80)
|
||
except OSError:
|
||
continue
|
||
|
||
|
||
def _image_size(path: Path | None) -> tuple[int | None, int | None]:
|
||
if path is None:
|
||
return None, None
|
||
try:
|
||
from PIL import Image
|
||
with Image.open(path) as image:
|
||
return image.width, image.height
|
||
except Exception:
|
||
return None, None
|
||
|
||
|
||
def _has_reconstruction_images(path: Path) -> bool:
|
||
if path.is_file():
|
||
return path.suffix.lower() in IMAGE_SUFFIXES
|
||
if not path.is_dir():
|
||
return False
|
||
return any(item.is_file() and item.suffix.lower() in IMAGE_SUFFIXES for item in path.iterdir())
|
||
|
||
|
||
def _prepared_manifest_for_dataset(dataset: dict[str, Any] | None) -> dict[str, Any]:
|
||
if not dataset:
|
||
return {}
|
||
manifest = _read_optional_json(Path(str(dataset.get("manifest_path") or "")))
|
||
if manifest.get("schema") == "real2sim.prepared_manifest.v1":
|
||
return manifest
|
||
path = Path(dataset["path"])
|
||
return _read_optional_json(path / "prepared_manifest.json")
|
||
|
||
|
||
def _prepared_manifest_path_for_input(path: Path) -> Path:
|
||
return path / "prepared_manifest.json" if path.is_dir() else path.parent / "prepared_manifest.json"
|
||
|
||
|
||
def _dataset_path(store: PlatformStore, params: dict[str, Any]) -> Path:
|
||
if params.get("dataset_path"):
|
||
return Path(str(params["dataset_path"]))
|
||
if params.get("dataset_id"):
|
||
return Path(store.get_dataset(str(params["dataset_id"]))["path"])
|
||
raise ValueError("training requires dataset_id or dataset_path")
|
||
|
||
|
||
def _scene_path(store: PlatformStore, params: dict[str, Any]) -> Path | None:
|
||
if params.get("scene_path"):
|
||
return Path(str(params["scene_path"]))
|
||
if params.get("scene_id"):
|
||
return Path(store.get_scene(str(params["scene_id"]))["path"])
|
||
return None
|
||
|
||
|
||
def _resolve_rm65_runtime_scene(store: PlatformStore, params: dict[str, Any], fallback: Path) -> Path:
|
||
candidates: list[Path] = []
|
||
if fallback.suffix.lower() == ".xml":
|
||
candidates.append(fallback)
|
||
if params.get("scene_id"):
|
||
scene = store.get_scene(str(params["scene_id"]))
|
||
metadata = scene.get("metadata") or {}
|
||
package = Path(str(metadata.get("package") or ""))
|
||
manifest_path = Path(str(scene.get("manifest_path") or ""))
|
||
manifest = _read_optional_json(manifest_path)
|
||
for asset in manifest.get("assets", []) if isinstance(manifest.get("assets"), list) else []:
|
||
if not isinstance(asset, dict):
|
||
continue
|
||
name = str(asset.get("name") or asset.get("path") or "").lower()
|
||
if "rm65" in name and name.endswith(".xml"):
|
||
candidates.append(Path(str(asset.get("path") or "")))
|
||
if str(package):
|
||
candidates.extend([
|
||
package / "simulator_assets" / "rm65_table_pick_place.xml",
|
||
package / "scene_mujoco.xml",
|
||
package / "mjcf" / "scene.xml",
|
||
])
|
||
candidates.extend([fallback.parent / "simulator_assets" / "rm65_table_pick_place.xml", fallback.parent / "scene_mujoco.xml"])
|
||
for candidate in candidates:
|
||
if candidate.is_file():
|
||
return candidate
|
||
raise FileNotFoundError("所选场景没有可执行的 RM65 pick-place MJCF;请先在场景管理中绑定 RM65 控制器和碰撞资产")
|
||
|
||
|
||
def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path:
|
||
if params.get("model_path"):
|
||
return Path(str(params["model_path"]))
|
||
if params.get("model_id"):
|
||
return Path(store.get_model(str(params["model_id"]))["path"])
|
||
raise ValueError("inference requires model_id or model_path")
|
||
|
||
|
||
def _directory_digest(root: Path) -> str:
|
||
digest = hashlib.sha256()
|
||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||
relative = path.relative_to(root).as_posix()
|
||
digest.update(relative.encode("utf-8"))
|
||
with path.open("rb") as handle:
|
||
while chunk := handle.read(8 * 1024 * 1024):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def _upload_directory(storage: BosStorage, root: Path, *, area: str, relative_root: str) -> None:
|
||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||
storage.upload_file(
|
||
path,
|
||
area=area,
|
||
relative_path=f"{relative_root}/{path.relative_to(root).as_posix()}",
|
||
metadata={"sha256": _file_sha256(path)},
|
||
)
|
||
|
||
|
||
def _file_sha256(path: Path) -> str:
|
||
digest = hashlib.sha256()
|
||
with path.open("rb") as handle:
|
||
while chunk := handle.read(8 * 1024 * 1024):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def _git_head(root: Path) -> str:
|
||
return read_source_revision(root)
|
||
|
||
|
||
def _inspect_act_dataset(dataset_path: Path) -> dict[str, Any]:
|
||
try:
|
||
import h5py
|
||
except ImportError as exc:
|
||
raise RuntimeError("DISCOVERSE ACT training requires h5py") from exc
|
||
episodes = sorted(dataset_path.glob("episode_*.hdf5"))
|
||
if not episodes:
|
||
raise ValueError("训练数据目录中没有 episode_*.hdf5")
|
||
lengths: list[int] = []
|
||
cameras: list[str] = []
|
||
state_dim = 0
|
||
action_dim = 0
|
||
action_alignment = ""
|
||
control_hold_steps = 0
|
||
for path in episodes:
|
||
with h5py.File(path, "r") as root:
|
||
if "/observations/qpos" not in root or "/action" not in root or "/observations/images" not in root:
|
||
raise ValueError(f"ACT HDF5 缺少 qpos/action/images: {path.name}")
|
||
current_cameras = sorted(str(name) for name in root["/observations/images"].keys())
|
||
current_state_dim = int(root["/observations/qpos"].shape[1])
|
||
current_action_dim = int(root["/action"].shape[1])
|
||
current_alignment = str(root.attrs.get("action_alignment", ""))
|
||
current_hold_steps = int(root.attrs.get("control_hold_steps", 0))
|
||
if current_alignment != "observation_t_to_action_t" or current_hold_steps <= 0:
|
||
raise ValueError(f"ACT HDF5 不是因果 observation_t -> action_t 数据: {path.name}")
|
||
if not cameras:
|
||
cameras = current_cameras
|
||
state_dim = current_state_dim
|
||
action_dim = current_action_dim
|
||
action_alignment = current_alignment
|
||
control_hold_steps = current_hold_steps
|
||
if (
|
||
current_cameras != cameras
|
||
or current_state_dim != state_dim
|
||
or current_action_dim != action_dim
|
||
or current_alignment != action_alignment
|
||
or current_hold_steps != control_hold_steps
|
||
):
|
||
raise ValueError("ACT 数据集的相机、状态维度、动作维度或控制周期不一致")
|
||
length = int(root["/action"].shape[0])
|
||
if length < 2:
|
||
raise ValueError(f"ACT episode 过短: {path.name}")
|
||
lengths.append(length)
|
||
return {
|
||
"episode_count": len(episodes),
|
||
"episode_length": min(lengths),
|
||
"camera_names": cameras,
|
||
"state_dim": state_dim,
|
||
"action_dim": action_dim,
|
||
"action_alignment": action_alignment,
|
||
"control_hold_steps": control_hold_steps,
|
||
}
|
||
|
||
|
||
def _act_task_config_source(
|
||
*,
|
||
task_name: str,
|
||
dataset_path: Path,
|
||
checkpoint_dir: Path,
|
||
stats_path: Path,
|
||
camera_names: list[str],
|
||
state_dim: int,
|
||
action_dim: int,
|
||
epochs: int,
|
||
batch_size: int,
|
||
chunk_size: int,
|
||
validate_every: int,
|
||
save_every: int,
|
||
) -> str:
|
||
return f'''from configurations.task_configs.template import TASK_CONFIG_DEFAULT, activator, set_paths
|
||
|
||
|
||
def policy_maker(config: dict, stage=None):
|
||
from policies.act.act import ACTPolicy
|
||
from policies.common.maker import post_init_policies
|
||
policy = ACTPolicy(config)
|
||
post_init_policies([policy], stage, [config["ckpt_path"]])
|
||
return policy
|
||
|
||
|
||
@activator(True)
|
||
def augment_images(image):
|
||
return image[:, [2, 1, 0], ...]
|
||
|
||
|
||
set_paths({str(dataset_path)!r}, {str(checkpoint_dir)!r}, {str(stats_path)!r})
|
||
TASK_CONFIG_DEFAULT["common"]["camera_names"] = {camera_names!r}
|
||
TASK_CONFIG_DEFAULT["common"]["state_dim"] = {state_dim}
|
||
TASK_CONFIG_DEFAULT["common"]["action_dim"] = {action_dim}
|
||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["policy_maker"] = policy_maker
|
||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["temporal_agg"] = True
|
||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["chunk_size"] = {chunk_size}
|
||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["num_queries"] = {chunk_size}
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["data_type"] = "hdf5"
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_episodes"] = "ALL"
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_train"] = {batch_size}
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_validate"] = {batch_size}
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_train"] = 2
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_validate"] = 2
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["observation_slice"] = None
|
||
TASK_CONFIG_DEFAULT["train"]["load_data"]["action_slice"] = None
|
||
TASK_CONFIG_DEFAULT["train"]["num_epochs"] = {epochs}
|
||
TASK_CONFIG_DEFAULT["train"]["validate_every"] = {validate_every}
|
||
TASK_CONFIG_DEFAULT["train"]["save_every"] = {save_every}
|
||
TASK_CONFIG_DEFAULT["train"]["eval_every"] = 0
|
||
TASK_CONFIG = TASK_CONFIG_DEFAULT
|
||
'''
|
||
|
||
|
||
def _parse_best_loss(description: str) -> float | None:
|
||
import re
|
||
|
||
match = re.search(r"val loss\s+([0-9.eE+-]+)", description)
|
||
if not match:
|
||
return None
|
||
try:
|
||
return float(match.group(1))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _write_dry_run_scene(package_dir: Path, input_path: Path, renderer: str, simulators: list[str]) -> Path:
|
||
package_dir.mkdir(parents=True, exist_ok=True)
|
||
scene = {
|
||
"schema": "discoverse.real2sim.scene_builder.v1",
|
||
"id": "dry-run-scene",
|
||
"name": "dry-run scene",
|
||
"coordinate_frame": "gs_metric_zup",
|
||
"source": str(input_path),
|
||
"renderer": renderer,
|
||
"simulators": simulators,
|
||
"assets": [],
|
||
"simulator_artifacts": _simulator_artifact_plan(package_dir, simulators),
|
||
}
|
||
path = package_dir / "scene.json"
|
||
path.write_text(json.dumps(scene, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
if "mujoco" in simulators:
|
||
(package_dir / "scene_mujoco.xml").write_text('<mujoco model="dry_run"><worldbody /></mujoco>\n', encoding="utf-8")
|
||
if "motrixsim" in simulators:
|
||
(package_dir / "motrix_import.json").write_text(json.dumps({"schema": "motrixsim.scene.v1", "source": str(input_path), "assets": []}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
(package_dir / "viewer.html").write_text("<!doctype html><title>dry-run scene</title><main>dry-run scene</main>\n", encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def _write_scene_manifest(package_dir: Path, *, input_path: Path, renderer: str, simulators: list[str], dry_run: bool, external_assets: list[dict[str, Any]]) -> Path:
|
||
package_dir.mkdir(parents=True, exist_ok=True)
|
||
assets = enrich_scene_manifest_assets({"assets": external_assets}).get("assets", [])
|
||
for path in _candidate_asset_files(package_dir):
|
||
rel = path.relative_to(package_dir)
|
||
assets.append(_viewer_asset(path, url=_run_static_url(path), role="package"))
|
||
manifest = {
|
||
"schema": "real2sim.scene_manifest.v1",
|
||
"input_path": str(input_path),
|
||
"renderer": renderer,
|
||
"renderer_status": renderer_status(renderer, package_dir),
|
||
"simulators": simulators,
|
||
"dry_run": dry_run,
|
||
"assets": assets,
|
||
"simulator_artifacts": _simulator_artifact_plan(package_dir, simulators),
|
||
}
|
||
path = package_dir / "scene_manifest.json"
|
||
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def enrich_scene_manifest_assets(manifest: dict[str, Any]) -> dict[str, Any]:
|
||
if not isinstance(manifest, dict) or not isinstance(manifest.get("assets"), list):
|
||
return manifest
|
||
enriched = dict(manifest)
|
||
assets = []
|
||
for item in manifest["assets"]:
|
||
if not isinstance(item, dict):
|
||
assets.append(item)
|
||
continue
|
||
asset = dict(item)
|
||
path_text = str(asset.get("path") or "")
|
||
path = Path(path_text) if path_text else None
|
||
if path and path.exists():
|
||
asset.setdefault("name", path.name)
|
||
asset.setdefault("format", path.suffix.lower().lstrip("."))
|
||
asset.setdefault("kind", _asset_kind(path))
|
||
size_bytes = path.stat().st_size
|
||
asset.setdefault("size_bytes", size_bytes)
|
||
asset.setdefault("size_label", _format_bytes(size_bytes))
|
||
else:
|
||
size_bytes = int(asset.get("size_bytes") or 0)
|
||
if size_bytes:
|
||
asset.setdefault("size_label", _format_bytes(size_bytes))
|
||
kind = str(asset.get("kind") or "")
|
||
autoload_default = kind in {"splat_or_pointcloud", "mesh"} and size_bytes <= VIEWER_AUTOLOAD_MAX_BYTES
|
||
asset.setdefault("autoload", autoload_default)
|
||
if "viewer_hint" not in asset:
|
||
if kind in {"splat_or_pointcloud", "mesh"} and size_bytes > VIEWER_AUTOLOAD_MAX_BYTES:
|
||
asset["viewer_hint"] = f"manual_load_large_asset_over_{VIEWER_AUTOLOAD_MAX_BYTES}_bytes"
|
||
elif autoload_default:
|
||
asset["viewer_hint"] = "autoload"
|
||
else:
|
||
asset["viewer_hint"] = "manual_load"
|
||
assets.append(asset)
|
||
enriched["assets"] = assets
|
||
return enriched
|
||
|
||
|
||
def renderer_status(renderer: str, package_dir: Path) -> dict[str, Any]:
|
||
aholo_manifest = package_dir / "aholo" / "manifest.json"
|
||
spark_available = SPARK_VENDOR_PATH.is_file()
|
||
three_available = THREE_VENDOR_PATH.is_file()
|
||
selected_backend = "aholo" if renderer == "aholo" and aholo_manifest.is_file() else "spark" if renderer == "spark" and spark_available else "three_fallback"
|
||
return {
|
||
"selected": renderer,
|
||
"backend": selected_backend,
|
||
"spark_available": spark_available,
|
||
"spark_bundle": str(SPARK_VENDOR_PATH),
|
||
"three_available": three_available,
|
||
"three_bundle": str(THREE_VENDOR_PATH),
|
||
"aholo_available": aholo_manifest.is_file(),
|
||
"aholo_reason": "Aholo LOD package found" if aholo_manifest.is_file() else "Aholo unavailable: LOD package is not present.",
|
||
"fallback_reason": "" if spark_available else "Three.js fallback: Spark bundle is not installed." if three_available else "Viewer unavailable: Three.js bundle is not installed.",
|
||
}
|
||
|
||
|
||
def _normalize_reconstruction_quality(value: str) -> str:
|
||
mapping = {
|
||
"discoverse-high": "capture-high",
|
||
"capture-high": "capture-high",
|
||
"high": "capture-high",
|
||
"capture-fast": "balanced",
|
||
"fast": "balanced",
|
||
"capture-debug": "balanced",
|
||
"debug": "balanced",
|
||
"motrix-high": "motrix-high",
|
||
"balanced": "balanced",
|
||
}
|
||
return mapping.get(value.strip().lower(), "capture-high")
|
||
|
||
|
||
def _append_discoverse_reconstruction_flags(command: list[str], *, params: dict[str, Any], quality_preset: str) -> None:
|
||
matcher = str(params.get("matcher") or params.get("sfm_matcher") or "exhaustive").strip().lower()
|
||
if matcher not in {"exhaustive", "sequential"}:
|
||
matcher = "exhaustive" if quality_preset == "capture-high" else "sequential"
|
||
command.extend(["--sfm-matcher", matcher])
|
||
camera_model = str(params.get("camera_model") or params.get("sfm_camera_model") or "auto").strip()
|
||
if camera_model and camera_model.lower() != "auto":
|
||
command.extend(["--sfm-camera-model", camera_model])
|
||
for param_name, flag_name in [
|
||
("max_image_size", "--sfm-max-image-size"),
|
||
("sift_max_num_features", "--sfm-sift-max-num-features"),
|
||
("sfm_sequential_overlap", "--sfm-sequential-overlap"),
|
||
("sfm_feature_threads", "--sfm-feature-threads"),
|
||
("sfm_matcher_threads", "--sfm-matcher-threads"),
|
||
("min_sfm_registered_ratio", "--min-sfm-registered-ratio"),
|
||
("min_sfm_points", "--min-sfm-points"),
|
||
("anysplat_max_views", "--anysplat-max-views"),
|
||
("anysplat_mesh_grid", "--anysplat-mesh-grid"),
|
||
("pose_init_train_steps", "--pose-init-train-steps"),
|
||
("pose_init_train_max_frames", "--pose-init-train-max-frames"),
|
||
("pose_init_train_image_max_size", "--pose-init-train-image-max-size"),
|
||
("pose_init_train_target_gaussians", "--pose-init-train-target-gaussians"),
|
||
("pose_init_train_max_points", "--pose-init-train-max-points"),
|
||
("pose_init_train_dense_image_points_per_frame", "--pose-init-train-dense-image-points-per-frame"),
|
||
]:
|
||
value = params.get(param_name)
|
||
if value not in (None, ""):
|
||
command.extend([flag_name, str(value)])
|
||
if bool(params.get("sfm_guided_matching", quality_preset == "capture-high")):
|
||
command.append("--sfm-guided-matching")
|
||
sfm_frames = str(params.get("sfm_frames") or ("selected" if quality_preset == "capture-high" else "extracted"))
|
||
if sfm_frames in {"extracted", "selected"}:
|
||
command.extend(["--sfm-frames", sfm_frames])
|
||
keyframe_strategy = str(params.get("anysplat_keyframe_strategy") or ("sfm-registered" if quality_preset == "capture-high" else "quality-diversity"))
|
||
if keyframe_strategy in {"quality-diversity", "sfm-registered"}:
|
||
command.extend(["--anysplat-keyframe-strategy", keyframe_strategy])
|
||
if quality_preset == "capture-high":
|
||
defaults = {
|
||
"--anysplat-max-views": "80",
|
||
"--pose-init-train-steps": "15000",
|
||
"--pose-init-train-target-gaussians": "1500000",
|
||
"--pose-init-train-max-points": "2000000",
|
||
"--pose-init-train-image-max-size": "1600",
|
||
"--pose-init-train-preview-frames": "16",
|
||
}
|
||
existing = set(command[1::2])
|
||
for flag, value in defaults.items():
|
||
if flag not in command:
|
||
command.extend([flag, value])
|
||
if "--pose-init-train-replace-scene" not in command and bool(params.get("pose_init_train_replace_scene", True)):
|
||
command.append("--pose-init-train-replace-scene")
|
||
if "--colmap-dense" not in command and bool(params.get("colmap_dense", False)):
|
||
command.append("--colmap-dense")
|
||
if "--colmap-texture-bake" not in command and bool(params.get("colmap_texture_bake", False)):
|
||
command.append("--colmap-texture-bake")
|
||
|
||
|
||
def _postprocess_discoverse_scene_assets(package_dir: Path, work_dir: Path, emit: Callable[[str], None]) -> dict[str, Any]:
|
||
report: dict[str, Any] = {
|
||
"schema": "real2sim.discoverse_scene_postprocess.v1",
|
||
"status": "not_run",
|
||
"exports": [],
|
||
"warnings": [],
|
||
}
|
||
source = _best_full_3dgs_asset(package_dir)
|
||
if source is None:
|
||
report["warnings"].append("no full 3DGS PLY with f_dc/opactity fields found; keeping original package assets")
|
||
_write_discoverse_postprocess_report(work_dir, report)
|
||
return report
|
||
target_dir = package_dir / "fallback" / "point_cloud"
|
||
target = target_dir / f"{source.stem}_threejs_rgb.ply"
|
||
export_report_path = target_dir / f"{source.stem}_threejs_rgb_report.json"
|
||
try:
|
||
export_report = _export_3dgs_rgb_fallback(source, target, export_report_path)
|
||
report["status"] = "succeeded"
|
||
report["source"] = str(source)
|
||
report["exports"].append(export_report)
|
||
emit(f"DISCOVERSE-style RGB fallback exported: {target.name} ({export_report.get('vertices', 0)} points)")
|
||
except Exception as exc:
|
||
report["status"] = "warning"
|
||
report["source"] = str(source)
|
||
report["warnings"].append(f"failed to export RGB fallback from 3DGS PLY: {type(exc).__name__}: {exc}")
|
||
_write_discoverse_postprocess_report(work_dir, report)
|
||
return report
|
||
|
||
|
||
def _write_discoverse_postprocess_report(work_dir: Path, report: dict[str, Any]) -> None:
|
||
work_dir.mkdir(parents=True, exist_ok=True)
|
||
(work_dir / "discoverse_postprocess_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
|
||
def _best_full_3dgs_asset(package_dir: Path) -> Path | None:
|
||
candidates = sorted(package_dir.rglob("*.ply"), key=_scene_asset_sort_key)
|
||
for path in candidates:
|
||
try:
|
||
header = _parse_ply_header(path)
|
||
except Exception:
|
||
continue
|
||
names = {name for _type, name in header["properties"]}
|
||
if {"x", "y", "z", "f_dc_0", "f_dc_1", "f_dc_2", "opacity"}.issubset(names):
|
||
return path
|
||
return None
|
||
|
||
|
||
def _parse_ply_header(path: Path) -> dict[str, Any]:
|
||
properties: list[tuple[str, str]] = []
|
||
vertex_count: int | None = None
|
||
fmt = ""
|
||
with path.open("rb") as handle:
|
||
first = handle.readline().decode("ascii", errors="replace").strip()
|
||
if first != "ply":
|
||
raise ValueError("not a PLY file")
|
||
while True:
|
||
raw = handle.readline()
|
||
if not raw:
|
||
raise ValueError("PLY header ended unexpectedly")
|
||
line = raw.decode("ascii", errors="replace").strip()
|
||
if line.startswith("format "):
|
||
fmt = line.split()[1]
|
||
elif line.startswith("element vertex "):
|
||
vertex_count = int(line.split()[-1])
|
||
elif line.startswith("property "):
|
||
parts = line.split()
|
||
if len(parts) >= 3:
|
||
properties.append((parts[-2], parts[-1]))
|
||
elif line == "end_header":
|
||
if vertex_count is None:
|
||
raise ValueError("PLY header missing vertex count")
|
||
return {"format": fmt, "vertex_count": vertex_count, "properties": properties, "data_offset": handle.tell()}
|
||
|
||
|
||
def _export_3dgs_rgb_fallback(source: Path, target: Path, report_path: Path) -> dict[str, Any]:
|
||
header = _parse_ply_header(source)
|
||
if header["format"] != "binary_little_endian":
|
||
raise ValueError(f"unsupported PLY format for exact RGB fallback: {header['format']}")
|
||
properties: list[tuple[str, str]] = header["properties"]
|
||
type_map = {
|
||
"char": "b",
|
||
"uchar": "B",
|
||
"int8": "b",
|
||
"uint8": "B",
|
||
"short": "h",
|
||
"ushort": "H",
|
||
"int16": "h",
|
||
"uint16": "H",
|
||
"int": "i",
|
||
"uint": "I",
|
||
"int32": "i",
|
||
"uint32": "I",
|
||
"float": "f",
|
||
"float32": "f",
|
||
"double": "d",
|
||
"float64": "d",
|
||
}
|
||
try:
|
||
input_struct = struct.Struct("<" + "".join(type_map[prop_type] for prop_type, _name in properties))
|
||
except KeyError as exc:
|
||
raise ValueError(f"unsupported PLY property type: {exc}") from exc
|
||
output_struct = struct.Struct("<fffBBB")
|
||
names = [name for _type, name in properties]
|
||
indexes = {name: index for index, name in enumerate(names)}
|
||
required = {"x", "y", "z", "f_dc_0", "f_dc_1", "f_dc_2", "opacity"}
|
||
missing = sorted(required.difference(indexes))
|
||
if missing:
|
||
raise ValueError(f"missing required 3DGS fields: {missing}")
|
||
count = int(header["vertex_count"])
|
||
bbox_min = [float("inf"), float("inf"), float("inf")]
|
||
bbox_max = [float("-inf"), float("-inf"), float("-inf")]
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
sh_c0 = 0.28209479177387814
|
||
with source.open("rb") as src, target.open("wb") as dst:
|
||
src.seek(int(header["data_offset"]))
|
||
dst.write((
|
||
"ply\n"
|
||
"format binary_little_endian 1.0\n"
|
||
"comment exported_by real2sim DISCOVERSE-style exact 3DGS RGB fallback\n"
|
||
f"comment source {source.as_posix()}\n"
|
||
f"element vertex {count}\n"
|
||
"property float x\n"
|
||
"property float y\n"
|
||
"property float z\n"
|
||
"property uchar red\n"
|
||
"property uchar green\n"
|
||
"property uchar blue\n"
|
||
"end_header\n"
|
||
).encode("ascii"))
|
||
for _ in range(count):
|
||
raw = src.read(input_struct.size)
|
||
if len(raw) != input_struct.size:
|
||
raise ValueError("PLY data ended before declared vertex count")
|
||
values = input_struct.unpack(raw)
|
||
x, y, z = values[indexes["x"]], values[indexes["y"]], values[indexes["z"]]
|
||
opacity = _sigmoid(float(values[indexes["opacity"]]))
|
||
rgb = []
|
||
for channel in ("f_dc_0", "f_dc_1", "f_dc_2"):
|
||
linear = float(values[indexes[channel]]) * sh_c0 + 0.5
|
||
linear = max(0.0, min(1.0, linear * min(1.0, opacity + 0.35)))
|
||
rgb.append(int(round(linear * 255)))
|
||
dst.write(output_struct.pack(float(x), float(y), float(z), rgb[0], rgb[1], rgb[2]))
|
||
for axis, value in enumerate((float(x), float(y), float(z))):
|
||
bbox_min[axis] = min(bbox_min[axis], value)
|
||
bbox_max[axis] = max(bbox_max[axis], value)
|
||
report = {
|
||
"schema": "real2sim.discoverse_exact_3dgs_rgb_fallback.v1",
|
||
"input": str(source),
|
||
"output": str(target),
|
||
"vertices": count,
|
||
"source_bbox_min": bbox_min,
|
||
"source_bbox_max": bbox_max,
|
||
"target_bbox_min": bbox_min,
|
||
"target_bbox_max": bbox_max,
|
||
"scale_xyz": [1.0, 1.0, 1.0],
|
||
"translate_xyz": [0.0, 0.0, 0.0],
|
||
"note": "RGB fallback point cloud exported from the exact same 3DGS PLY DC color fields so visual layers stay aligned.",
|
||
}
|
||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
return report
|
||
|
||
|
||
def _sigmoid(value: float) -> float:
|
||
if value >= 0:
|
||
z = math.exp(-value)
|
||
return 1.0 / (1.0 + z)
|
||
z = math.exp(value)
|
||
return z / (1.0 + z)
|
||
|
||
|
||
def _add_preview_asset_artifacts(store: PlatformStore, task_id: str, input_path: Path) -> list[dict[str, Any]]:
|
||
assets: list[dict[str, Any]] = []
|
||
for path in _candidate_asset_files(input_path)[:12]:
|
||
artifact = store.add_artifact(task_id=task_id, name=path.name, path=path, kind=path.suffix.lower().lstrip(".") or "file")
|
||
assets.append(_viewer_asset(path, url=f"/api/artifacts/{artifact['id']}/content", role="source"))
|
||
return assets
|
||
|
||
|
||
def _viewer_asset(path: Path, *, url: str, role: str) -> dict[str, Any]:
|
||
size_bytes = path.stat().st_size if path.exists() else 0
|
||
suffix = path.suffix.lower().lstrip(".")
|
||
kind = _asset_kind(path)
|
||
autoload = kind in {"splat_or_pointcloud", "mesh"} and size_bytes <= VIEWER_AUTOLOAD_MAX_BYTES
|
||
hint = "autoload" if autoload else "manual_load"
|
||
if kind in {"splat_or_pointcloud", "mesh"} and size_bytes > VIEWER_AUTOLOAD_MAX_BYTES:
|
||
hint = f"manual_load_large_asset_over_{VIEWER_AUTOLOAD_MAX_BYTES}_bytes"
|
||
return {
|
||
"name": path.name,
|
||
"path": str(path),
|
||
"url": url,
|
||
"kind": kind,
|
||
"role": role,
|
||
"format": suffix,
|
||
"size_bytes": size_bytes,
|
||
"size_label": _format_bytes(size_bytes),
|
||
"autoload": autoload,
|
||
"viewer_hint": hint,
|
||
}
|
||
|
||
|
||
def _candidate_asset_files(root: Path) -> list[Path]:
|
||
if root.is_file():
|
||
files = [root]
|
||
elif root.is_dir():
|
||
files = [path for path in root.rglob("*") if path.is_file()]
|
||
else:
|
||
return []
|
||
return sorted([path for path in files if path.suffix.lower() in SCENE_ASSET_SUFFIXES], key=_scene_asset_sort_key)
|
||
|
||
|
||
def _scene_asset_sort_key(path: Path) -> tuple[int, int, str]:
|
||
name = path.name.lower()
|
||
full = path.as_posix().lower()
|
||
if name == "scene_cover.png":
|
||
rank = 0
|
||
elif name.endswith("_threejs_rgb.ply") or "fallback/point_cloud" in full:
|
||
rank = 1
|
||
elif name.endswith((".splat", ".spz", ".ksplat", ".sog")):
|
||
rank = 2
|
||
elif name == "pose_init_trained_scene.ply":
|
||
rank = 3
|
||
elif name == "scene.ply" or full.endswith("/3dgs/scene.ply"):
|
||
rank = 4
|
||
elif name in {"pointcloud_projection_preview.png", "source_gs_preview.png"}:
|
||
rank = 5
|
||
elif name.startswith("thumb_") and path.suffix.lower() in {".png", ".jpg", ".jpeg"}:
|
||
rank = 6
|
||
elif name.endswith(".ply") and ("colmap_sparse" in name or "sparse_points" in name):
|
||
rank = 10
|
||
elif name.endswith(".ply"):
|
||
rank = 7
|
||
elif path.suffix.lower() in {".obj", ".glb", ".gltf"}:
|
||
rank = 8
|
||
elif path.suffix.lower() == ".xml":
|
||
rank = 9
|
||
elif path.suffix.lower() == ".json":
|
||
rank = 11
|
||
else:
|
||
rank = 12
|
||
size = path.stat().st_size if path.exists() else 0
|
||
return (rank, -size, name)
|
||
|
||
|
||
def _asset_kind(path: Path) -> str:
|
||
suffix = path.suffix.lower()
|
||
if suffix in SPARK_ASSET_SUFFIXES:
|
||
return "splat_or_pointcloud"
|
||
if suffix in {".obj", ".glb", ".gltf"}:
|
||
return "mesh"
|
||
if suffix == ".xml":
|
||
return "mujoco"
|
||
if suffix == ".json":
|
||
return "manifest"
|
||
if suffix in {".png", ".jpg", ".jpeg"}:
|
||
return "image"
|
||
return "file"
|
||
|
||
|
||
def _format_bytes(size: int) -> str:
|
||
value = float(size)
|
||
for unit in ["B", "KB", "MB", "GB"]:
|
||
if value < 1024 or unit == "GB":
|
||
return f"{value:.1f} {unit}" if unit != "B" else f"{int(value)} B"
|
||
value /= 1024
|
||
return f"{size} B"
|
||
|
||
|
||
def _run_static_url(path: Path) -> str:
|
||
parts = path.parts
|
||
if "runs" in parts:
|
||
index = parts.index("runs")
|
||
return "/" + "/".join(parts[index:])
|
||
return str(path)
|
||
|
||
|
||
def _scene_manifest_path_for_scene(scene: dict[str, Any]) -> Path:
|
||
manifest_path = Path(str(scene.get("manifest_path") or ""))
|
||
if manifest_path.is_file():
|
||
return manifest_path
|
||
scene_path = Path(str(scene.get("path") or ""))
|
||
package_dir = scene_path if scene_path.is_dir() else scene_path.parent
|
||
candidate = package_dir / "scene_manifest.json"
|
||
if candidate.is_file():
|
||
return candidate
|
||
raise FileNotFoundError(f"scene manifest not found for scene path {scene_path}")
|
||
|
||
|
||
def _find_chromium_binary() -> Path | None:
|
||
candidates = [
|
||
os.environ.get("REAL2SIM_CHROME_BIN", ""),
|
||
"/usr/bin/google-chrome",
|
||
"/usr/bin/google-chrome-stable",
|
||
"/usr/bin/chromium",
|
||
"/usr/bin/chromium-browser",
|
||
"/snap/bin/chromium",
|
||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||
]
|
||
for item in candidates:
|
||
if item and Path(item).is_file():
|
||
return Path(item)
|
||
for name in ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]:
|
||
found = shutil.which(name)
|
||
if found:
|
||
return Path(found)
|
||
return None
|
||
|
||
|
||
def _render_scene_cover_candidates(config: PlatformConfig, chrome: Path, scene_id: str, artifacts_dir: Path, emit: Callable[[str], None]) -> list[dict[str, Any]]:
|
||
candidates: list[dict[str, Any]] = []
|
||
for view in ["hero", "front", "side", "top"]:
|
||
out_path = artifacts_dir / f"scene_cover_{view}.png"
|
||
url = f"http://127.0.0.1:{config.port}/static/scene-viewer.html?scene_id={scene_id}&cover=1&cover_view={view}"
|
||
command = [
|
||
str(chrome),
|
||
"--headless=new",
|
||
"--disable-gpu",
|
||
"--no-sandbox",
|
||
"--hide-scrollbars",
|
||
"--window-size=1400,820",
|
||
"--virtual-time-budget=12000",
|
||
f"--screenshot={out_path}",
|
||
url,
|
||
]
|
||
emit("cover command: " + " ".join(command))
|
||
try:
|
||
completed = subprocess.run(command, cwd=str(config.app_root), text=True, capture_output=True, timeout=45)
|
||
except Exception as exc:
|
||
emit(f"cover view {view} failed: {exc}")
|
||
continue
|
||
if completed.returncode != 0 or not out_path.is_file() or out_path.stat().st_size < 4096:
|
||
reason = (completed.stderr or completed.stdout or f"chromium exited {completed.returncode}").strip()
|
||
emit(f"cover view {view} failed: {reason}")
|
||
continue
|
||
score_info = _score_cover_image(out_path)
|
||
candidates.append({
|
||
"view": view,
|
||
"path": str(out_path),
|
||
"url": _run_static_url(out_path),
|
||
"size_bytes": out_path.stat().st_size,
|
||
**score_info,
|
||
})
|
||
return candidates
|
||
|
||
|
||
def _score_cover_image(path: Path) -> dict[str, Any]:
|
||
size_score = min(path.stat().st_size / 180000.0, 1.2)
|
||
try:
|
||
from PIL import Image, ImageStat # type: ignore
|
||
|
||
image = Image.open(path).convert("RGB").resize((160, 94))
|
||
pixels = list(image.getdata())
|
||
non_dark = sum(1 for r, g, b in pixels if r + g + b > 48)
|
||
non_dark_ratio = non_dark / max(len(pixels), 1)
|
||
stat = ImageStat.Stat(image)
|
||
variance = sum(stat.var) / 3.0
|
||
variance_score = min(variance / 1800.0, 1.4)
|
||
score = non_dark_ratio * 2.6 + variance_score + size_score
|
||
return {
|
||
"score": round(score, 4),
|
||
"non_dark_ratio": round(non_dark_ratio, 4),
|
||
"variance": round(variance, 2),
|
||
}
|
||
except Exception:
|
||
return {"score": round(size_score, 4), "non_dark_ratio": None, "variance": None}
|
||
|
||
|
||
def _copy_existing_scene_preview(manifest: dict[str, Any], cover_path: Path) -> None:
|
||
assets = [asset for asset in manifest.get("assets", []) if isinstance(asset, dict)]
|
||
image_assets = [
|
||
asset for asset in assets
|
||
if str(asset.get("path") or "").lower().endswith((".png", ".jpg", ".jpeg", ".webp"))
|
||
or str(asset.get("name") or "").lower().endswith((".png", ".jpg", ".jpeg", ".webp"))
|
||
]
|
||
image_assets.sort(key=lambda asset: _scene_preview_rank(str(asset.get("name") or asset.get("path") or "")))
|
||
for asset in image_assets:
|
||
path = Path(str(asset.get("path") or ""))
|
||
if path.is_file():
|
||
shutil.copy2(path, cover_path)
|
||
return
|
||
|
||
|
||
def _scene_preview_rank(name: str) -> tuple[int, str]:
|
||
lowered = name.lower()
|
||
if "scene_cover" in lowered:
|
||
return (0, lowered)
|
||
if "pointcloud_projection_preview" in lowered:
|
||
return (1, lowered)
|
||
if "source_gs_preview" in lowered:
|
||
return (2, lowered)
|
||
if "preview" in lowered:
|
||
return (3, lowered)
|
||
if "background_image" in lowered:
|
||
return (4, lowered)
|
||
if "thumb" in lowered:
|
||
return (5, lowered)
|
||
return (9, lowered)
|
||
|
||
|
||
def _simulator_artifact_plan(package_dir: Path, simulators: list[str]) -> dict[str, Any]:
|
||
result: dict[str, Any] = {}
|
||
if "mujoco" in simulators:
|
||
result["mujoco"] = {"expected": str(package_dir / "scene_mujoco.xml")}
|
||
if "motrixsim" in simulators:
|
||
result["motrixsim"] = {"expected": str(package_dir / "motrix_import.json")}
|
||
return result
|
||
|
||
|
||
def _validate_sim_asset(candidates: list[Path], *, required_suffix: str) -> dict[str, Any]:
|
||
existing = next((path for path in candidates if path.is_file() and path.suffix.lower() == required_suffix), None)
|
||
if existing is None:
|
||
return {"ok": False, "path": None, "error": f"no {required_suffix} simulator artifact found"}
|
||
text = existing.read_text(encoding="utf-8", errors="ignore")[:20000]
|
||
checks = {
|
||
"exists": True,
|
||
"non_empty": existing.stat().st_size > 0,
|
||
"has_coordinate_or_schema": any(token in text.lower() for token in ["schema", "coordinate", "mujoco", "motrix", "worldbody"]),
|
||
}
|
||
return {"ok": all(checks.values()), "path": str(existing), "checks": checks}
|
||
|
||
|
||
def _validate_mujoco_asset(candidates: list[Path], *, level: str) -> dict[str, Any]:
|
||
result = _validate_sim_asset(candidates, required_suffix=".xml")
|
||
if not result["ok"] or level == "smoke":
|
||
result["level_effective"] = "smoke"
|
||
return result
|
||
try:
|
||
import mujoco # type: ignore
|
||
except Exception as exc:
|
||
result["runtime"] = {"status": "unavailable", "error": str(exc)}
|
||
result["level_effective"] = "smoke"
|
||
result["ok"] = result["ok"] and level != "runtime_required"
|
||
return result
|
||
try:
|
||
model = mujoco.MjModel.from_xml_path(str(result["path"]))
|
||
result["runtime"] = {"status": "loaded", "nbody": int(model.nbody), "ngeom": int(model.ngeom)}
|
||
result["level_effective"] = "runtime"
|
||
result["ok"] = True
|
||
except Exception as exc:
|
||
result["runtime"] = {"status": "failed", "error": str(exc)}
|
||
result["level_effective"] = "runtime"
|
||
result["ok"] = False
|
||
return result
|
||
|
||
|
||
def _validate_motrixsim_asset(candidates: list[Path], *, level: str) -> dict[str, Any]:
|
||
descriptor = _validate_sim_asset(candidates, required_suffix=".json")
|
||
if level == "smoke":
|
||
descriptor["level_effective"] = "smoke"
|
||
return descriptor
|
||
runtime_scene = next((path for path in candidates if path.is_file() and path.suffix.lower() == ".xml"), None)
|
||
if runtime_scene is None:
|
||
return {
|
||
"ok": False,
|
||
"path": descriptor.get("path"),
|
||
"descriptor": descriptor,
|
||
"level_effective": "runtime",
|
||
"runtime": {"status": "artifact_unavailable", "error": "no MJCF scene is available for MotrixSim runtime loading"},
|
||
}
|
||
try:
|
||
from motrixsim import SceneData, load_model, step # type: ignore
|
||
except Exception as exc:
|
||
return {
|
||
"ok": level != "runtime_required" and bool(descriptor.get("ok")),
|
||
"path": str(runtime_scene),
|
||
"descriptor": descriptor,
|
||
"level_effective": "runtime_if_available",
|
||
"runtime": {"status": "unavailable", "error": str(exc), "tooling": _motrixsim_tooling()},
|
||
}
|
||
try:
|
||
model = load_model(str(runtime_scene))
|
||
data = SceneData(model)
|
||
for _ in range(8):
|
||
step(model, data)
|
||
return {
|
||
"ok": True,
|
||
"path": str(runtime_scene),
|
||
"descriptor": descriptor,
|
||
"level_effective": "runtime",
|
||
"runtime": {
|
||
"status": "loaded_and_stepped",
|
||
"steps": 8,
|
||
"timestep": float(model.options.timestep),
|
||
"nq": len(data.low.dof_positions),
|
||
"nv": len(data.low.dof_velocities),
|
||
},
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"ok": False,
|
||
"path": str(runtime_scene),
|
||
"descriptor": descriptor,
|
||
"level_effective": "runtime",
|
||
"runtime": {"status": "failed", "error": f"{type(exc).__name__}: {exc}"},
|
||
}
|
||
|
||
|
||
def _motrixsim_tooling() -> dict[str, Any]:
|
||
candidates = [
|
||
Path("/opt/motrixsim-docs"),
|
||
Path("/opt/Motphys/motrixsim-docs/examples/hello_motrixsim.py"),
|
||
Path("/opt/Motphys/MotrixLab"),
|
||
Path("/opt/Motphys/motrixsim_playground"),
|
||
]
|
||
existing = [str(path) for path in candidates if path.exists()]
|
||
return {"available": bool(existing), "paths": existing}
|
||
|
||
|
||
def _find_scene_json(package_dir: Path) -> Path | None:
|
||
for name in ["scene.json", "motrix_import.json", "package_manifest.json"]:
|
||
candidate = package_dir / name
|
||
if candidate.is_file():
|
||
return candidate
|
||
matches = sorted(package_dir.rglob("scene.json"))
|
||
return matches[0] if matches else None
|
||
|
||
|
||
def _index_artifacts(store: PlatformStore, task_id: str, root: Path) -> None:
|
||
indexable_suffixes = {
|
||
".json", ".jsonl", ".html", ".svg", ".png", ".jpg", ".jpeg", ".mp4", ".zip", ".tar", ".gz",
|
||
".ply", ".spz", ".splat", ".ksplat", ".sog", ".rad", ".obj", ".glb", ".gltf", ".xml",
|
||
".hdf5", ".h5", ".ckpt", ".pkl", ".npy", ".npz", ".csv", ".yaml", ".yml", ".toml", ".py",
|
||
".log", ".txt",
|
||
}
|
||
for path in sorted(root.rglob("*")):
|
||
if not path.is_file():
|
||
continue
|
||
if path.suffix.lower() in indexable_suffixes:
|
||
store.add_artifact(task_id=task_id, name=path.name, path=path, kind=path.suffix.lower().lstrip(".") or "file")
|
||
|
||
|
||
def index_artifacts_for_task(store: PlatformStore, task_id: str, root: Path) -> None:
|
||
_index_artifacts(store, task_id, root)
|
||
|
||
|
||
def collect_task_reports(store: PlatformStore, task_id: str) -> dict[str, Any]:
|
||
task = store.get_task(task_id)
|
||
run_dir = Path(task["run_dir"])
|
||
report_names = [
|
||
"capture_preflight_report.json",
|
||
"capture_acceptance_report.json",
|
||
"reconstruction_input.json",
|
||
"environment.json",
|
||
"reconstruction_command.json",
|
||
"reconstruction_report.json",
|
||
"validation_report.json",
|
||
"motrixsim_asset_import_report.json",
|
||
"splat_optimization_report.json",
|
||
"prepared_manifest.json",
|
||
"dataset_manifest.json",
|
||
"model_card.json",
|
||
]
|
||
reports: list[dict[str, Any]] = []
|
||
task_error = str(task.get("error") or "")
|
||
report_errors: list[tuple[int, str]] = []
|
||
candidates = list(sorted(run_dir.rglob("*.json")) if run_dir.exists() else [])
|
||
for artifact in store.list_task_artifacts(task_id):
|
||
artifact_path = Path(artifact["path"])
|
||
if artifact_path.suffix.lower() == ".json" and artifact_path not in candidates:
|
||
candidates.append(artifact_path)
|
||
for path in candidates:
|
||
if path.name not in report_names:
|
||
continue
|
||
payload = _read_optional_json(path)
|
||
failure_summary = _report_failure_summary(path.name, payload)
|
||
if failure_summary is not None:
|
||
report_errors.append(failure_summary)
|
||
try:
|
||
relative_path = str(path.relative_to(run_dir))
|
||
except ValueError:
|
||
relative_path = path.name
|
||
reports.append(
|
||
{
|
||
"name": path.name,
|
||
"path": str(path),
|
||
"relative_path": relative_path,
|
||
"payload": payload,
|
||
}
|
||
)
|
||
return {
|
||
"task_id": task_id,
|
||
"type": task["type"],
|
||
"status": task["status"],
|
||
"readable_error": _choose_readable_error(task_error, report_errors),
|
||
"reports": reports,
|
||
}
|
||
|
||
|
||
def _choose_readable_error(task_error: str, report_errors: list[tuple[int, str]]) -> str:
|
||
clean = task_error.strip()
|
||
if report_errors:
|
||
report_error = max(report_errors, key=lambda item: item[0])[1]
|
||
if not clean or _is_generic_task_error(clean):
|
||
return report_error
|
||
return clean
|
||
|
||
|
||
def _is_generic_task_error(value: str) -> bool:
|
||
clean = value.strip().lower()
|
||
return clean.startswith(("command failed with exit code", "subprocess exited with", "process failed with exit code"))
|
||
|
||
|
||
def _report_failure_summary(name: str, payload: dict[str, Any]) -> tuple[int, str] | None:
|
||
if not isinstance(payload, dict) or not payload:
|
||
return None
|
||
schema = str(payload.get("schema") or "")
|
||
status = str(payload.get("status") or "").lower()
|
||
if name == "capture_acceptance_report.json" or "capture_acceptance" in schema:
|
||
blockers = _string_items(payload.get("blockers") or payload.get("failed_gates"))
|
||
decision = str(payload.get("decision") or payload.get("reason") or "").strip()
|
||
accepted = payload.get("accepted")
|
||
if accepted is False or blockers or status in {"failed", "not_run", "rejected"}:
|
||
parts = ["capture acceptance failed"]
|
||
if decision:
|
||
parts[0] += f": {decision}"
|
||
if blockers:
|
||
parts.append("blockers=" + ", ".join(blockers))
|
||
return (100, "; ".join(parts))
|
||
if name == "capture_preflight_report.json" or "capture_preflight" in schema:
|
||
failed_gates = _string_items(payload.get("failed_gates"))
|
||
if failed_gates:
|
||
return (90, "reconstruction preflight failed: " + ", ".join(failed_gates))
|
||
message = str(payload.get("readable_error") or payload.get("error") or "").strip()
|
||
if message:
|
||
return (80, message)
|
||
if name == "validation_report.json" or "simulator_validation" in schema:
|
||
if payload.get("ok") is False:
|
||
failures = []
|
||
simulators = payload.get("simulators") or {}
|
||
if isinstance(simulators, dict):
|
||
for simulator, result in simulators.items():
|
||
if isinstance(result, dict) and not result.get("ok", False):
|
||
reason = str(result.get("reason") or result.get("status") or "failed")
|
||
failures.append(f"{simulator}:{reason}")
|
||
return (80, "simulator validation failed: " + (", ".join(failures) if failures else "one or more simulators failed"))
|
||
if name == "reconstruction_report.json" or "reconstruction_report" in schema:
|
||
if status == "failed":
|
||
message = str(payload.get("readable_error") or payload.get("error") or "").strip()
|
||
if message:
|
||
return (70, message)
|
||
if payload.get("ok") is False or status in {"failed", "not_run", "rejected"}:
|
||
message = str(payload.get("readable_error") or payload.get("error") or payload.get("reason") or "").strip()
|
||
if message:
|
||
return (50, message)
|
||
return None
|
||
|
||
|
||
def _string_items(value: Any) -> list[str]:
|
||
if value is None or value == "":
|
||
return []
|
||
if isinstance(value, str):
|
||
return [value]
|
||
if isinstance(value, dict):
|
||
items = []
|
||
for key, item in value.items():
|
||
if isinstance(item, bool):
|
||
if item:
|
||
items.append(str(key))
|
||
elif item not in (None, ""):
|
||
items.append(f"{key}={item}")
|
||
return items
|
||
if isinstance(value, list):
|
||
items = []
|
||
for item in value:
|
||
if isinstance(item, dict):
|
||
label = item.get("id") or item.get("name") or item.get("check") or item.get("label") or item.get("reason")
|
||
if label:
|
||
items.append(str(label))
|
||
elif item not in (None, ""):
|
||
items.append(str(item))
|
||
return items
|
||
return [str(value)]
|
||
|
||
|
||
def _normalize_patterns(value: Any) -> list[str]:
|
||
if value is None or value == "":
|
||
return []
|
||
if isinstance(value, str):
|
||
return [part.strip() for part in value.split(",") if part.strip()]
|
||
if isinstance(value, list):
|
||
return [str(part).strip() for part in value if str(part).strip()]
|
||
raise ValueError("allow_patterns must be a string or list")
|
||
|
||
|
||
def _safe_name(value: str) -> str:
|
||
clean = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "-" for ch in value).strip(".-")
|
||
return clean or "hf-dataset"
|
||
|
||
|
||
def _file_inventory(root: Path, *, limit: int) -> list[dict[str, Any]]:
|
||
files = [path for path in root.rglob("*") if path.is_file()]
|
||
items = []
|
||
for path in sorted(files)[:limit]:
|
||
items.append({"path": str(path.relative_to(root)), "size": path.stat().st_size, "kind": _asset_kind(path)})
|
||
return items
|
||
|
||
|
||
def _env_with_paths(config: PlatformConfig, python_paths: list[Path] | None = None) -> dict[str, str]:
|
||
env = os.environ.copy()
|
||
runtime_paths = [
|
||
config.anysplat_root,
|
||
config.runtime_root / "speedy-splat",
|
||
config.runtime_root / "sam2",
|
||
config.runtime_root / "sam-3d-objects",
|
||
config.runtime_root / "sam-3d-objects" / "notebook",
|
||
]
|
||
paths = [str(path) for path in [*(python_paths or []), *runtime_paths] if path.exists()]
|
||
if paths:
|
||
existing = env.get("PYTHONPATH", "")
|
||
env["PYTHONPATH"] = os.pathsep.join(paths + ([existing] if existing else []))
|
||
env.setdefault("REAL2SIM_COLMAP_BIN", str(config.colmap_binary))
|
||
env.setdefault("REAL2SIM_ANYSPLAT_ROOT", str(config.anysplat_root))
|
||
env.setdefault("REAL2SIM_ANY_SPLAT_PYTHON", str(config.anysplat_python))
|
||
env.setdefault("REAL2SIM_POSE_INIT_TRAIN_PYTHON", str(config.gsplat_python))
|
||
env.setdefault("CUDA_HOME", str(config.cuda_home))
|
||
cuda_bin = config.cuda_home / "bin"
|
||
cuda_lib = config.cuda_home / "lib64"
|
||
if cuda_bin.is_dir():
|
||
env["PATH"] = str(cuda_bin) + os.pathsep + env.get("PATH", "")
|
||
if cuda_lib.is_dir():
|
||
env["LD_LIBRARY_PATH"] = str(cuda_lib) + os.pathsep + env.get("LD_LIBRARY_PATH", "")
|
||
return env
|
||
|
||
|
||
def _read_json(path: Path) -> dict[str, Any]:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def _read_optional_json(path: Path) -> dict[str, Any]:
|
||
try:
|
||
if path.is_file():
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
return {}
|
||
return {}
|
||
|
||
|
||
def _tail_text(path: Path, *, max_chars: int = 4000) -> str:
|
||
try:
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
except OSError:
|
||
return ""
|
||
return text[-max_chars:]
|
||
|
||
|
||
def _readable_failure_from_log(text: str) -> str:
|
||
fallback = ""
|
||
for line in reversed(text.splitlines()):
|
||
clean = line.strip()
|
||
if not clean or clean.startswith(("Traceback", "File ", "During handling of the above exception")):
|
||
continue
|
||
for prefix in ["ERROR:", "RuntimeError:", "ValueError:"]:
|
||
if clean.startswith(prefix):
|
||
clean = clean[len(prefix):].strip()
|
||
if clean.startswith("$ "):
|
||
continue
|
||
if clean.startswith(("raise ", "subprocess.CalledProcessError", "CalledProcessError(")):
|
||
continue
|
||
if not fallback:
|
||
fallback = clean
|
||
lowered = clean.lower()
|
||
if any(marker in lowered for marker in [" requires ", " failed", "error", "missing", "not found", "recapture", "blocker"]):
|
||
return clean
|
||
return fallback
|
||
|
||
|
||
def _append_log(path: Path, message: str) -> None:
|
||
path.write_text(path.read_text(encoding="utf-8") + message.rstrip() + "\n", encoding="utf-8")
|
||
|
||
|
||
def _module_available(name: str) -> bool:
|
||
try:
|
||
import importlib.util
|
||
return importlib.util.find_spec(name) is not None
|
||
except (ImportError, ValueError):
|
||
return False
|
||
|
||
|
||
@lru_cache(maxsize=1)
|
||
def _torch_cuda_ready() -> bool:
|
||
if not _module_available("torch"):
|
||
return False
|
||
completed = subprocess.run(
|
||
[sys.executable, "-c", "import torch; raise SystemExit(0 if torch.cuda.is_available() else 2)"],
|
||
text=True,
|
||
capture_output=True,
|
||
timeout=30,
|
||
check=False,
|
||
)
|
||
return completed.returncode == 0
|
||
|
||
|
||
@lru_cache(maxsize=4)
|
||
def _act_runtime_import_ready(discoverse_root_value: str, discoverse_commit: str, act_commit: str) -> bool:
|
||
discoverse_root = Path(discoverse_root_value)
|
||
act_root = discoverse_root / "policies" / "act"
|
||
if _git_head(discoverse_root) != discoverse_commit or _git_head(act_root) != act_commit or not _torch_cuda_ready():
|
||
return False
|
||
env = os.environ.copy()
|
||
env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep)
|
||
completed = subprocess.run(
|
||
[sys.executable, "-c", "from policies.act.act import ACTPolicy; assert ACTPolicy is not None"],
|
||
cwd=act_root,
|
||
env=env,
|
||
text=True,
|
||
capture_output=True,
|
||
timeout=45,
|
||
check=False,
|
||
)
|
||
return completed.returncode == 0
|
||
|
||
|
||
def _is_writable(path: Path) -> bool:
|
||
try:
|
||
path.mkdir(parents=True, exist_ok=True)
|
||
probe = path / ".write_probe"
|
||
probe.write_text("ok", encoding="utf-8")
|
||
probe.unlink()
|
||
return True
|
||
except OSError:
|
||
return False
|