feat: productize Real2Sim for isolated POC cluster

This commit is contained in:
eust-w 2026-07-10 23:59:43 +08:00
parent ad41f0fe72
commit d3c5b397d7
79 changed files with 28575 additions and 2133 deletions

11
.dockerignore Normal file
View File

@ -0,0 +1,11 @@
.git
.venv
.pytest_cache
workspace
acceptance_runs
qa
docs
tests
**/__pycache__
**/*.pyc
*.tgz

View File

@ -0,0 +1,324 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: real2sim-runtime-config
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
data:
config.toml: |
app_root = "/app"
workspace_dir = "/var/lib/real2sim-poc/workspace"
host = "0.0.0.0"
port = 18126
default_renderer = "spark"
real2sim2model_root = "/app"
expert_policy_root = "/app"
training_root = "/app"
inference_root = "/app"
discoverse_root = "/opt/DISCOVERSE"
gs_playground_root = "/opt/gs_playground"
images23dgs_root = "/opt/images23dgs"
colmap_binary = "/usr/bin/colmap"
gsplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python"
runtime_root = "/opt/real2sim_paper_runtime"
anysplat_root = "/opt/real2sim_paper_runtime/AnySplat"
anysplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python"
cuda_home = "/usr/local/cuda"
runtime_env_script = "/opt/gs_playground/scripts/real2sim_wuying_paper_env.sh"
allowed_path_roots = "/var/lib/real2sim-poc:/opt/DISCOVERSE:/opt/gs_playground:/opt/motrixsim-docs:/opt/real2sim_paper_runtime:/tmp"
production_mode = true
auth_enabled = true
secure_cookies = true
session_ttl_seconds = 43200
discoverse_commit = "d67f47c084aba0e0cf422a8725235f8b9238655a"
act_commit = "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"
anysplat_commit = "5f5e208a7dd57d52e43ea0d553a95eab526e8775"
anysplat_model_revision = "d2e8c343672646041ad4ea518184968f94362f01"
vggt_model_revision = "860abec7937da0a4c03c41d3c269c366e82abdf9"
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: real2sim-api
namespace: real2sim-poc
labels:
app.kubernetes.io/name: real2sim-api
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/version: "__BUILD_ID__"
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels:
app.kubernetes.io/name: real2sim-api
template:
metadata:
labels:
app.kubernetes.io/name: real2sim-api
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/version: "__BUILD_ID__"
spec:
serviceAccountName: real2sim-poc
automountServiceAccountToken: false
nodeSelector:
kubernetes.io/hostname: "192.168.80.136"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
imagePullSecrets:
- name: real2sim-registry
containers:
- name: api
image: __API_IMAGE__
imagePullPolicy: IfNotPresent
env:
- {name: REAL2SIM_PLATFORM_CONFIG, value: /etc/real2sim/config.toml}
- {name: REAL2SIM_WORKER_MODE, value: api}
- {name: REAL2SIM_WORKER_ID, value: real2sim-worker-a800}
- {name: REAL2SIM_PRODUCTION_MODE, value: "1"}
- {name: REAL2SIM_AUTH_ENABLED, value: "1"}
- {name: REAL2SIM_SECURE_COOKIES, value: "1"}
- {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc}
- {name: BOS_BUCKET, value: testxyz}
- {name: BOS_ENDPOINT, value: https://bj.bcebos.com}
- name: BOS_ACCESS_KEY_ID
valueFrom: {secretKeyRef: {name: real2sim-bos, key: accessKeyId}}
- name: BOS_SECRET_ACCESS_KEY
valueFrom: {secretKeyRef: {name: real2sim-bos, key: secretAccessKey}}
- name: REAL2SIM_BOOTSTRAP_ADMIN_PASSWORD
valueFrom: {secretKeyRef: {name: real2sim-app-secrets, key: bootstrap-admin-password}}
ports:
- {name: http, containerPort: 18126}
resources:
requests: {cpu: 500m, memory: 1Gi}
limits: {cpu: "4", memory: 8Gi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
readinessProbe:
httpGet: {path: /api/health, port: http}
initialDelaySeconds: 3
periodSeconds: 5
livenessProbe:
httpGet: {path: /api/health, port: http}
initialDelaySeconds: 20
periodSeconds: 20
volumeMounts:
- {name: data, mountPath: /var/lib/real2sim-poc}
- {name: config, mountPath: /etc/real2sim/config.toml, subPath: config.toml, readOnly: true}
- {name: tmp, mountPath: /tmp}
volumes:
- name: data
persistentVolumeClaim: {claimName: real2sim-poc-data}
- name: config
configMap: {name: real2sim-runtime-config}
- name: tmp
emptyDir: {sizeLimit: 2Gi}
---
apiVersion: v1
kind: Service
metadata:
name: real2sim-api
namespace: real2sim-poc
labels:
app.kubernetes.io/name: real2sim-api
app.kubernetes.io/part-of: real2sim-poc
spec:
selector:
app.kubernetes.io/name: real2sim-api
ports:
- {name: http, port: 18126, targetPort: http}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: real2sim-worker
namespace: real2sim-poc
labels:
app.kubernetes.io/name: real2sim-worker
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/version: "__BUILD_ID__"
spec:
replicas: 1
strategy: {type: Recreate}
selector:
matchLabels:
app.kubernetes.io/name: real2sim-worker
template:
metadata:
labels:
app.kubernetes.io/name: real2sim-worker
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/version: "__BUILD_ID__"
spec:
serviceAccountName: real2sim-poc
automountServiceAccountToken: false
nodeSelector:
kubernetes.io/hostname: "192.168.80.136"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
imagePullSecrets:
- name: real2sim-registry
initContainers:
- name: runtime-gate
image: __WORKER_IMAGE__
imagePullPolicy: IfNotPresent
command: [real2sim-runtime-gate, --mode, bootstrap]
env: &worker-env
- {name: REAL2SIM_PLATFORM_CONFIG, value: /etc/real2sim/config.toml}
- {name: REAL2SIM_WORKER_MODE, value: worker}
- {name: REAL2SIM_WORKER_ID, value: real2sim-worker-a800}
- {name: REAL2SIM_PRODUCTION_MODE, value: "1"}
- {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc}
- {name: REAL2SIM_RUNTIME_GATE_DIR, value: /var/lib/real2sim-poc/cache/runtime-gate}
- {name: HF_HOME, value: /var/lib/real2sim-poc/cache/huggingface}
- {name: HF_HUB_DISABLE_XET, value: "1"}
- {name: BOS_BUCKET, value: testxyz}
- {name: BOS_ENDPOINT, value: https://bj.bcebos.com}
- name: BOS_ACCESS_KEY_ID
valueFrom: {secretKeyRef: {name: real2sim-bos, key: accessKeyId}}
- name: BOS_SECRET_ACCESS_KEY
valueFrom: {secretKeyRef: {name: real2sim-bos, key: secretAccessKey}}
resources:
requests: {cpu: "6", memory: 32Gi, nvidia.com/gpu: "1"}
limits: {cpu: "24", memory: 96Gi, nvidia.com/gpu: "1"}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts: &worker-mounts
- {name: data, mountPath: /var/lib/real2sim-poc}
- {name: config, mountPath: /etc/real2sim/config.toml, subPath: config.toml, readOnly: true}
- {name: tmp, mountPath: /tmp}
- {name: shm, mountPath: /dev/shm}
containers:
- name: worker
image: __WORKER_IMAGE__
imagePullPolicy: IfNotPresent
env: *worker-env
resources:
requests: {cpu: "6", memory: 32Gi, nvidia.com/gpu: "1"}
limits: {cpu: "24", memory: 96Gi, nvidia.com/gpu: "1"}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
readinessProbe:
exec: {command: [real2sim-runtime-gate, --mode, readiness]}
initialDelaySeconds: 5
periodSeconds: 15
timeoutSeconds: 10
failureThreshold: 4
livenessProbe:
exec: {command: [real2sim-runtime-gate, --mode, readiness]}
initialDelaySeconds: 45
periodSeconds: 60
timeoutSeconds: 15
failureThreshold: 3
volumeMounts: *worker-mounts
volumes:
- name: data
persistentVolumeClaim: {claimName: real2sim-poc-data}
- name: config
configMap: {name: real2sim-runtime-config}
- name: tmp
emptyDir: {sizeLimit: 12Gi}
- name: shm
emptyDir: {medium: Memory, sizeLimit: 12Gi}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: real2sim-frontend
namespace: real2sim-poc
labels:
app.kubernetes.io/name: real2sim-frontend
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/version: "__BUILD_ID__"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: real2sim-frontend
template:
metadata:
labels:
app.kubernetes.io/name: real2sim-frontend
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/version: "__BUILD_ID__"
spec:
serviceAccountName: real2sim-poc
automountServiceAccountToken: false
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["192.168.80.249", "192.168.81.47"]
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
seccompProfile: {type: RuntimeDefault}
imagePullSecrets:
- name: real2sim-registry
containers:
- name: frontend
image: __FRONTEND_IMAGE__
imagePullPolicy: IfNotPresent
ports:
- {name: http, containerPort: 8080}
- {name: https, containerPort: 8443}
resources:
requests: {cpu: 100m, memory: 128Mi}
limits: {cpu: "1", memory: 512Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
readinessProbe:
tcpSocket: {port: https}
initialDelaySeconds: 3
periodSeconds: 5
livenessProbe:
tcpSocket: {port: https}
initialDelaySeconds: 15
periodSeconds: 20
volumeMounts:
- {name: tls, mountPath: /etc/real2sim/tls, readOnly: true}
- {name: tmp, mountPath: /tmp}
- {name: nginx-cache, mountPath: /var/cache/nginx}
volumes:
- name: tls
secret: {secretName: real2sim-tls}
- name: tmp
emptyDir: {sizeLimit: 64Mi}
- name: nginx-cache
emptyDir: {sizeLimit: 128Mi}
---
apiVersion: v1
kind: Service
metadata:
name: real2sim-frontend
namespace: real2sim-poc
labels:
app.kubernetes.io/name: real2sim-frontend
app.kubernetes.io/part-of: real2sim-poc
spec:
type: LoadBalancer
selector:
app.kubernetes.io/name: real2sim-frontend
ports:
- {name: http, port: 80, targetPort: http}
- {name: https, port: 443, targetPort: https}

184
deploy/poc/bootstrap.yaml Normal file
View File

@ -0,0 +1,184 @@
apiVersion: v1
kind: Namespace
metadata:
name: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: real2sim-poc-local-pv
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
capacity:
storage: 150Gi
volumeMode: Filesystem
accessModes: [ReadWriteOnce]
persistentVolumeReclaimPolicy: Retain
storageClassName: real2sim-poc-local
local:
path: /var/lib/real2sim-poc
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["192.168.80.136"]
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: real2sim-poc-data
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
accessModes: [ReadWriteOnce]
storageClassName: real2sim-poc-local
volumeName: real2sim-poc-local-pv
resources:
requests:
storage: 150Gi
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: real2sim-poc-quota
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
hard:
pods: "12"
services: "6"
services.loadbalancers: "1"
requests.cpu: "40"
limits.cpu: "64"
requests.memory: 128Gi
limits.memory: 192Gi
requests.storage: 150Gi
requests.nvidia.com/gpu: "1"
limits.nvidia.com/gpu: "1"
---
apiVersion: v1
kind: LimitRange
metadata:
name: real2sim-poc-limits
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
limits:
- type: Container
defaultRequest:
cpu: 100m
memory: 128Mi
default:
cpu: "2"
memory: 4Gi
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: real2sim-poc
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
automountServiceAccountToken: false
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
podSelector: {}
policyTypes: [Ingress, Egress]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-dns
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
podSelector: {}
policyTypes: [Egress]
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: frontend-policy
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: real2sim-frontend
policyTypes: [Ingress, Egress]
ingress:
- ports:
- {protocol: TCP, port: 8080}
- {protocol: TCP, port: 8443}
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: real2sim-api
ports:
- {protocol: TCP, port: 18126}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: api-policy
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: real2sim-api
policyTypes: [Ingress, Egress]
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: real2sim-frontend
ports:
- {protocol: TCP, port: 18126}
egress:
- ports:
- {protocol: TCP, port: 443}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: worker-policy
namespace: real2sim-poc
labels:
app.kubernetes.io/part-of: real2sim-poc
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: real2sim-worker
policyTypes: [Ingress, Egress]
egress:
- ports:
- {protocol: TCP, port: 443}
- {protocol: TCP, port: 80}

82
deploy/poc/nginx.conf Normal file
View File

@ -0,0 +1,82 @@
worker_processes auto;
pid /tmp/nginx.pid;
events { worker_connections 1024; }
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /dev/stdout;
error_log /dev/stderr warn;
sendfile on;
server_tokens off;
client_max_body_size 70m;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
upstream real2sim_api { server real2sim-api.real2sim-poc.svc.cluster.local:18126; }
server {
listen 8080;
return 308 https://$host$request_uri;
}
server {
listen 8443 ssl;
ssl_certificate /etc/real2sim/tls/tls.crt;
ssl_certificate_key /etc/real2sim/tls/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_cache shared:TLS:10m;
add_header X-Content-Type-Options nosniff always;
add_header Referrer-Policy same-origin always;
add_header X-Frame-Options SAMEORIGIN always;
root /usr/share/nginx/html;
index index.html;
location = /api/events/tasks {
proxy_pass http://real2sim_api;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto https;
proxy_buffering off;
proxy_cache off;
proxy_read_timeout 1h;
}
location /api/ {
proxy_pass http://real2sim_api;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_request_buffering off;
proxy_read_timeout 7200s;
}
location /datasets/ {
proxy_pass http://real2sim_api;
proxy_set_header Host $host;
}
location ~ ^/static/.*\.(html|js|css)$ {
try_files $uri =404;
expires -1;
add_header Cache-Control "no-store";
}
location /static/ {
try_files $uri =404;
expires 1h;
add_header Cache-Control "public, max-age=3600";
}
location / {
try_files $uri $uri/ /index.html;
add_header Cache-Control "no-store";
}
}
}

View File

@ -0,0 +1,45 @@
numpy==1.25.0
wheel
tqdm
lightning
hydra-core
jaxtyping
beartype
einops
colorama
scikit-image
colorspacious
matplotlib
moviepy==1.0.3
imageio
timm
dacite
lpips
e3nn
plyfile
tabulate
svg.py
scikit-video
opencv-python-headless==4.8.1.78
Pillow
huggingface_hub
pydantic
open3d
safetensors
omegaconf
pyquaternion
pyyaml
packaging
h5py
dm_env
OneEuroFilter
pymeshlab
scipy
psutil
screeninfo
mediapy
termcolor
imagecodecs
av
wandb
trimesh

18
docker/api.Dockerfile Normal file
View File

@ -0,0 +1,18 @@
FROM python:3.12-slim-bookworm
ARG BUILD_ID=dev
ARG SOURCE_COMMIT=unknown
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \
REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT}
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg libglib2.0-0 && rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY pyproject.toml README.md ./
COPY src ./src
RUN python -m pip install --no-cache-dir --upgrade pip && python -m pip install --no-cache-dir .
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim
USER 10001:10001
EXPOSE 18126
CMD ["real2sim-platform", "--host", "0.0.0.0", "--port", "18126"]

View File

@ -0,0 +1,10 @@
FROM nginxinc/nginx-unprivileged:1.27-alpine
ARG BUILD_ID=dev
LABEL org.opencontainers.image.title="Real2Sim POC frontend" \
org.opencontainers.image.version="${BUILD_ID}"
COPY deploy/poc/nginx.conf /etc/nginx/nginx.conf
COPY src/real2sim_platform/static/index.html /usr/share/nginx/html/index.html
COPY src/real2sim_platform/static /usr/share/nginx/html/static
USER 101:101
EXPOSE 8080 8443

View File

@ -0,0 +1,2 @@
numpy==1.25.0
opencv-python-headless==4.8.1.78

72
docker/worker.Dockerfile Normal file
View File

@ -0,0 +1,72 @@
FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime
ARG BUILD_ID=dev
ARG SOURCE_COMMIT=unknown
ARG ANYSPLAT_COMMIT=5f5e208a7dd57d52e43ea0d553a95eab526e8775
ARG DISCOVERSE_COMMIT=d67f47c084aba0e0cf422a8725235f8b9238655a
ARG ACT_COMMIT=f8ab39c8d829d85571ab6d0250412ba4d0698fa0
ARG MOTRIXSIM_DOCS_COMMIT=2beecea67c81d25e29cefa30a80c0ddad7b44dde
ARG VGGT_MODEL_REVISION=860abec7937da0a4c03c41d3c269c366e82abdf9
ENV DEBIAN_FRONTEND=noninteractive \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \
REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT} \
REAL2SIM_VGGT_MODEL_REVISION=${VGGT_MODEL_REVISION} \
REAL2SIM_ANYSPLAT_ROOT=/opt/real2sim_paper_runtime/AnySplat \
REAL2SIM_ANY_SPLAT_PYTHON=/opt/real2sim_paper_runtime/envs/anysplat/bin/python \
REAL2SIM_POSE_INIT_TRAIN_PYTHON=/opt/real2sim_paper_runtime/envs/anysplat/bin/python \
CUDA_HOME=/usr/local/cuda \
PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat:/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE:/opt/gs_playground
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl ffmpeg git build-essential cmake ninja-build pkg-config colmap \
libgl1 libglib2.0-0 libgomp1 libx11-6 libxext6 libsm6 && \
rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY pyproject.toml README.md ./
COPY src ./src
COPY scripts ./scripts
COPY vendor/gs_playground /opt/gs_playground
COPY docker/anysplat-runtime-requirements.txt /tmp/anysplat-runtime-requirements.txt
COPY docker/worker-constraints.txt /tmp/worker-constraints.txt
RUN python -m pip install --no-cache-dir --upgrade pip setuptools wheel && \
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt . && \
python -m pip install --no-cache-dir torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121 && \
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt -r /tmp/anysplat-runtime-requirements.txt && \
python -m pip install --no-cache-dir xformers==0.0.24 --no-deps && \
python -m pip install --no-cache-dir torch-scatter==2.1.2 -f https://data.pyg.org/whl/torch-2.2.0+cu121.html && \
python -m pip install --no-cache-dir 'https://github.com/nerfstudio-project/gsplat/releases/download/v1.4.0/gsplat-1.4.0%2Bpt22cu121-cp310-cp310-linux_x86_64.whl' && \
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt motrixsim-core==0.8.0
RUN git clone --filter=blob:none --no-checkout https://github.com/InternRobotics/AnySplat.git /opt/real2sim_paper_runtime/AnySplat && \
git -C /opt/real2sim_paper_runtime/AnySplat checkout "${ANYSPLAT_COMMIT}" && \
test "$(git -C /opt/real2sim_paper_runtime/AnySplat rev-parse HEAD)" = "${ANYSPLAT_COMMIT}" && \
sed -i 's/VGGT.from_pretrained("facebook\/VGGT-1B")/VGGT.from_pretrained("facebook\/VGGT-1B", revision=os.environ.get("REAL2SIM_VGGT_MODEL_REVISION"))/g' /opt/real2sim_paper_runtime/AnySplat/src/model/encoder/anysplat.py
RUN git clone --filter=blob:none --no-checkout https://github.com/eust-w/DISCOVERSE.git /opt/DISCOVERSE && \
git -C /opt/DISCOVERSE checkout "${DISCOVERSE_COMMIT}" && \
git -C /opt/DISCOVERSE submodule update --init policies/act && \
test "$(git -C /opt/DISCOVERSE rev-parse HEAD)" = "${DISCOVERSE_COMMIT}" && \
test "$(git -C /opt/DISCOVERSE/policies/act rev-parse HEAD)" = "${ACT_COMMIT}" && \
python -m pip install --no-cache-dir --no-deps -e /opt/DISCOVERSE
RUN git clone --filter=blob:none --no-checkout https://github.com/Motphys/motrixsim-docs.git /opt/motrixsim-docs && \
git -C /opt/motrixsim-docs checkout "${MOTRIXSIM_DOCS_COMMIT}" && \
test "$(git -C /opt/motrixsim-docs rev-parse HEAD)" = "${MOTRIXSIM_DOCS_COMMIT}" && \
mkdir -p /opt/real2sim_paper_runtime/envs/anysplat/bin && \
ln -s /opt/conda/bin/python /opt/real2sim_paper_runtime/envs/anysplat/bin/python && \
ln -s /opt/conda/bin/pip /opt/real2sim_paper_runtime/envs/anysplat/bin/pip
RUN python -c 'import mujoco, motrixsim, torch; from huggingface_hub import snapshot_download; assert torch.__version__.startswith("2.2")' && \
PYTHONPATH=/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE python -c 'from policy import ACTPolicy; assert ACTPolicy is not None' && \
PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat python -c 'from src.model.model.anysplat import AnySplat; assert AnySplat is not None' && \
test -f /opt/gs_playground/scripts/real2sim_video_motrix_scene.py
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim && \
chown -R 10001:10001 /app
USER 10001:10001
CMD ["real2sim-worker"]

45
docs/POC_PRODUCT_SPEC.md Normal file
View File

@ -0,0 +1,45 @@
# Real2Sim POC 产品与交互规格
## 用户与边界
| 角色 | 主要目标 | 写权限 |
| --- | --- | --- |
| 管理员 | 初始化平台、管理用户、存储和删除策略 | 全部;删除磁盘或 BOS 对象必须二次确认 |
| 算法工程师 | 导入数据、重建、采集、训练和推理 | 业务资源和任务,不可管理用户 |
| 只读用户 | 查看场景、报告、模型和 rollout | 无 |
平台面向内部 POC/Beta,不开放注册、计费或多组织租户。真实 MuJoCo 和 MotrixSim 是允许的领域运行时;禁止 mock、dry-run 验收、硬编码成功率、假进度和通过直接改物体位姿伪造抓取。
## 完整用户旅程
1. 管理员使用一次性初始密码登录,强制更新密码,再创建工程师或只读账号。
2. 工程师从 HuggingFace、浏览器分片上传或授权 BOS 前缀导入数据。系统计算 SHA-256、创建版本和清单。
3. 视频或图片包进入“准备输入”,统一生成 prepared frames;质量预检给出帧数、分辨率、清晰度、视差和可执行建议。
4. 场景重建向导选择 prepared dataset、质量档位、坐标系、渲染器和仿真器。任务立即进入本页,SSE 显示真实阶段和进度。
5. 重建成功后在场景管理中打开 Spark/Three Viewer,添加机器人或物体会生成新 revision,不覆盖来源场景。
6. 数据采集页只展示场景 manifest 声明兼容的真实控制器。RM65 采集要求双指接触、至少抬升 10 cm、放置误差不超过 5 cm。
7. 模型训练页只允许新建 DISCOVERSE ACT;配置由 HDF5 和场景 manifest 推导,输出 checkpoint、损失历史、模型卡和血缘。
8. 在线推理在真实引擎运行随机化 episode,输出视频、轨迹、动作统计和成功率。没有真实结果时显示空状态。
## 七页交互合同
- **总览**:当前闭环阶段、推荐下一步、真实任务、最近场景和模型结果。无运维状态条或调试 JSON。
- **数据管理**:来源卡打开导入弹窗;表格支持全选、多选、批量准备、预检和删除索引。
- **场景重建**:资源卡选择、质量评分、参数折叠、任务列表和完成后预览构成单页闭环。
- **场景管理**:场景库、3D Viewer、资产和仿真信息三栏;移动端按 Viewer、场景库、资产编辑顺序堆叠。
- **数据采集**:场景、控制器和采集规模在同页;任务完成后直接出现 HDF5 数据集。
- **模型训练**:采集数据、ACT 配方、真实指标、训练任务和模型卡同页呈现。
- **在线推理**:模型和场景卡选择、rollout 播放、轨迹产物与任务管理同页呈现。
任务详情使用右侧抽屉展示阶段、报告、建议、产物预览和下载;高级 JSON 仅在折叠区。删除默认只删平台索引,不删 PV 或 BOS 对象。
## 可行性证据与交付门槛
- 本地已用真实 MuJoCo 和 MotrixSim 加载同一 RM65 MJCF 并执行 step。
- 本地真实物理控制器已完成双指接触抓放:抬升 15.57 cm,放置误差 1.29 cm,输出 RGB/qpos/action/contact HDF5 与 MP4。
- 集群 worker 必须在 A800 上通过 CUDA 矩阵运算、AnySplat 固定 revision 模型加载、ACT import、MuJoCo step 和 MotrixSim step,之后才 Ready。
- 最终 `acceptance_report.json.pass=true` 还要求两次非 dry-run 重建、50 个成功采集 episode、真实 ACT checkpoint,以及 10 次推理成功率至少 50%。未达门槛时可保留 Beta 部署,但不得声明全链路通过。
## 视觉合同
视觉原型保存于 `docs/prototypes/real2sim-poc-product-board-v1.png`。实现采用深色中性底、青绿色命令色、紧凑表格、6px 面板圆角和纯文字导航;禁止大编号、运行时状态条、装饰图标、裸资源 ID/路径主输入及无数据时的虚构图表。

View File

@ -8,6 +8,8 @@ version = "0.1.0"
description = "Real2Sim scene, collection, training, and inference management platform"
requires-python = ">=3.10"
dependencies = [
"argon2-cffi>=23.1",
"bce-python-sdk>=0.9.29",
"fastapi>=0.110",
"huggingface_hub>=0.24",
"imageio>=2.34",
@ -22,6 +24,8 @@ dependencies = [
[project.scripts]
real2sim-platform = "real2sim_platform.server:main"
real2sim-worker = "real2sim_platform.worker_main:main"
real2sim-runtime-gate = "real2sim_platform.runtime_gate:main"
[tool.setuptools.packages.find]
where = ["src"]
@ -36,3 +40,6 @@ where = ["src"]
"static/vendor/three/addons/postprocessing/*",
"static/vendor/three/addons/utils/*",
]
[tool.pytest.ini_options]
markers = ["integration: exercises installed real runtimes and physical scene assets"]

205
scripts/deploy_poc_cluster.sh Executable file
View File

@ -0,0 +1,205 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}"
NAMESPACE="real2sim-poc"
GPU_NODE="${REAL2SIM_POC_GPU_NODE:-192.168.80.136}"
REGISTRY="${REAL2SIM_POC_REGISTRY:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc}"
SOURCE_COMMIT="$(git -C "$ROOT" rev-parse --short=12 HEAD)"
BUILD_ID="${REAL2SIM_POC_BUILD_ID:-${SOURCE_COMMIT}-$(date -u +%Y%m%d%H%M%S)}"
OUT="${REAL2SIM_POC_ACCEPTANCE_OUT:-$ROOT/acceptance_runs/poc/$BUILD_ID}"
BUILD_IMAGES=1
DEPLOY_APP=1
tls_dir=""
context_archive=""
rendered_manifest=""
trap 'rm -rf "${tls_dir:-}" "${context_archive:-}" "${rendered_manifest:-}"' EXIT
for arg in "$@"; do
case "$arg" in
--skip-build) BUILD_IMAGES=0 ;;
--build-only) DEPLOY_APP=0 ;;
--help)
echo "usage: $0 [--skip-build|--build-only]"
exit 0
;;
*) echo "unknown argument: $arg" >&2; exit 2 ;;
esac
done
K=(kubectl --kubeconfig "$KUBECONFIG_PATH")
mkdir -p "$OUT"
require() { command -v "$1" >/dev/null || { echo "missing required command: $1" >&2; exit 1; }; }
for command in kubectl jq openssl tar git sed; do require "$command"; done
test -f "$KUBECONFIG_PATH" || { echo "kubeconfig not found: $KUBECONFIG_PATH" >&2; exit 1; }
test "$(git -C "$ROOT" branch --show-current)" = "feat/poc-cluster-productization" || {
echo "deployment is only allowed from feat/poc-cluster-productization" >&2
exit 1
}
gpu_capacity="$("${K[@]}" get node "$GPU_NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}')"
gpu_in_use="$("${K[@]}" get pods -A --field-selector "spec.nodeName=$GPU_NODE" -o json | jq '[.items[] | select(.status.phase=="Running" or .status.phase=="Pending") | ([.spec.containers[]?.resources.requests["nvidia.com/gpu"] // "0" | tonumber] | add // 0)] | add // 0')"
if (( gpu_capacity - gpu_in_use < 1 )); then
echo "A800 node has no free GPU: capacity=$gpu_capacity in_use=$gpu_in_use" >&2
exit 1
fi
printf 'gpu_capacity=%s\ngpu_in_use=%s\nbuild_id=%s\n' "$gpu_capacity" "$gpu_in_use" "$BUILD_ID" > "$OUT/deploy_preflight.txt"
"${K[@]}" apply -f "$ROOT/deploy/poc/bootstrap.yaml"
clone_secret() {
local source_namespace="$1" source_name="$2" target_name="$3"
"${K[@]}" -n "$source_namespace" get secret "$source_name" -o json |
jq --arg ns "$NAMESPACE" --arg name "$target_name" '
del(.metadata.uid,.metadata.resourceVersion,.metadata.creationTimestamp,.metadata.managedFields,.metadata.ownerReferences) |
.metadata.namespace=$ns | .metadata.name=$name |
.metadata.labels={"app.kubernetes.io/part-of":"real2sim-poc"}' |
"${K[@]}" apply -f - >/dev/null
}
clone_secret droboticflow droboticflow-registry real2sim-registry
clone_secret droboticflow droboticflow-bos-workspace real2sim-bos
if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets >/dev/null 2>&1; then
admin_password="$(openssl rand -base64 30 | tr -d '/+=' | cut -c1-28)Aa1!"
session_secret="$(openssl rand -hex 32)"
"${K[@]}" -n "$NAMESPACE" create secret generic real2sim-app-secrets \
--from-literal=bootstrap-admin-password="$admin_password" \
--from-literal=session-secret="$session_secret" \
--dry-run=client -o yaml |
"${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml |
"${K[@]}" apply -f - >/dev/null
fi
if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-tls >/dev/null 2>&1; then
tls_dir="$(mktemp -d)"
openssl req -x509 -newkey rsa:3072 -sha256 -nodes -days 365 \
-subj "/CN=real2sim-poc.local" \
-addext "subjectAltName=DNS:real2sim-poc.local,DNS:localhost,IP:127.0.0.1" \
-keyout "$tls_dir/tls.key" -out "$tls_dir/tls.crt" >/dev/null 2>&1
"${K[@]}" -n "$NAMESPACE" create secret tls real2sim-tls \
--key "$tls_dir/tls.key" --cert "$tls_dir/tls.crt" --dry-run=client -o yaml |
"${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml |
"${K[@]}" apply -f - >/dev/null
fi
"${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets -o jsonpath='{.data.bootstrap-admin-password}' |
base64 --decode > "$OUT/bootstrap-admin-password.txt"
chmod 600 "$OUT/bootstrap-admin-password.txt"
printf 'admin\n' > "$OUT/bootstrap-admin-username.txt"
chmod 600 "$OUT/bootstrap-admin-username.txt"
prepare_context() {
context_archive="$(mktemp -t real2sim-poc-context.XXXXXX.tgz)"
tar -C "$ROOT" -czf "$context_archive" \
--exclude='.git' --exclude='.venv' --exclude='.pytest_cache' --exclude='workspace' \
--exclude='acceptance_runs' --exclude='qa' --exclude='docs' --exclude='tests' .
local loader="real2sim-context-${BUILD_ID//[^a-zA-Z0-9-]/-}"
loader="${loader:0:60}"
"${K[@]}" -n "$NAMESPACE" delete pod "$loader" --ignore-not-found --wait=true >/dev/null
"${K[@]}" -n "$NAMESPACE" run "$loader" \
--image=busybox:1.36 --restart=Never \
--labels="app.kubernetes.io/part-of=real2sim-poc,app.kubernetes.io/component=build-context" \
--overrides="$(jq -cn --arg node "$GPU_NODE" '{spec:{nodeSelector:{"kubernetes.io/hostname":$node},containers:[{name:"context-loader",image:"busybox:1.36",command:["sh","-c","sleep 3600"],resources:{requests:{cpu:"100m",memory:"64Mi"},limits:{cpu:"500m",memory:"256Mi"}},volumeMounts:[{name:"data",mountPath:"/data"}]}],volumes:[{name:"data",persistentVolumeClaim:{claimName:"real2sim-poc-data"}}]}}')" >/dev/null
"${K[@]}" -n "$NAMESPACE" wait --for=condition=Ready "pod/$loader" --timeout=180s
"${K[@]}" -n "$NAMESPACE" cp "$context_archive" "$loader:/tmp/context.tgz"
"${K[@]}" -n "$NAMESPACE" exec "$loader" -- sh -eu -c \
"mkdir -p '/data/build-context/$BUILD_ID' /data/workspace /data/cache /data/scratch; rm -rf '/data/build-context/$BUILD_ID/'*; tar -xzf /tmp/context.tgz -C '/data/build-context/$BUILD_ID'; chown -R 10001:10001 /data/workspace /data/cache /data/scratch; chmod 0770 /data/workspace /data/cache /data/scratch"
"${K[@]}" -n "$NAMESPACE" delete pod "$loader" --wait=true >/dev/null
}
build_image() {
local component="$1" dockerfile="$2" destination="$3"
local job="real2sim-build-${component}-${BUILD_ID//[^a-zA-Z0-9-]/-}"
job="${job:0:60}"
"${K[@]}" -n "$NAMESPACE" delete job "$job" --ignore-not-found --wait=true >/dev/null
cat <<YAML | "${K[@]}" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
name: $job
namespace: $NAMESPACE
labels:
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/component: image-build
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/component: image-build
spec:
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: "$GPU_NODE"
containers:
- name: kaniko
image: gcr.io/kaniko-project/executor:v1.23.2-debug
args:
- --context=dir:///data/build-context/$BUILD_ID
- --dockerfile=/data/build-context/$BUILD_ID/$dockerfile
- --destination=$destination
- --build-arg=BUILD_ID=$BUILD_ID
- --build-arg=SOURCE_COMMIT=$SOURCE_COMMIT
- --snapshot-mode=redo
- --use-new-run
- --cache=true
- --cache-repo=$REGISTRY/real2sim-build-cache
- --verbosity=info
resources:
requests: {cpu: "2", memory: 4Gi}
limits: {cpu: "16", memory: 32Gi}
volumeMounts:
- {name: data, mountPath: /data}
- {name: docker-config, mountPath: /kaniko/.docker, readOnly: true}
volumes:
- name: data
persistentVolumeClaim: {claimName: real2sim-poc-data}
- name: docker-config
secret:
secretName: real2sim-registry
items:
- {key: .dockerconfigjson, path: config.json}
YAML
if ! "${K[@]}" -n "$NAMESPACE" wait --for=condition=complete "job/$job" --timeout=7200s; then
"${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log" 2>&1 || true
"${K[@]}" -n "$NAMESPACE" describe "job/$job" > "$OUT/build-${component}-describe.txt" 2>&1 || true
return 1
fi
"${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log"
"${K[@]}" -n "$NAMESPACE" delete job "$job" --wait=true >/dev/null
}
FRONTEND_IMAGE="$REGISTRY/real2sim-frontend:$BUILD_ID"
API_IMAGE="$REGISTRY/real2sim-api:$BUILD_ID"
WORKER_IMAGE="$REGISTRY/real2sim-worker:$BUILD_ID"
printf '%s\n' "$FRONTEND_IMAGE" "$API_IMAGE" "$WORKER_IMAGE" > "$OUT/images.txt"
if [[ "$BUILD_IMAGES" == "1" ]]; then
prepare_context
build_image frontend docker/frontend.Dockerfile "$FRONTEND_IMAGE"
build_image api docker/api.Dockerfile "$API_IMAGE"
build_image worker docker/worker.Dockerfile "$WORKER_IMAGE"
fi
if [[ "$DEPLOY_APP" == "1" ]]; then
rendered_manifest="$(mktemp -t real2sim-poc-manifest.XXXXXX.yaml)"
sed \
-e "s|__BUILD_ID__|$BUILD_ID|g" \
-e "s|__FRONTEND_IMAGE__|$FRONTEND_IMAGE|g" \
-e "s|__API_IMAGE__|$API_IMAGE|g" \
-e "s|__WORKER_IMAGE__|$WORKER_IMAGE|g" \
"$ROOT/deploy/poc/app-template.yaml" > "$rendered_manifest"
"${K[@]}" apply -f "$rendered_manifest"
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-api --timeout=600s
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-frontend --timeout=600s
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-worker --timeout=3600s
"${K[@]}" -n "$NAMESPACE" get pods,svc,pvc -l app.kubernetes.io/part-of=real2sim-poc -o wide > "$OUT/resources.txt"
"${K[@]}" -n "$NAMESPACE" get svc real2sim-frontend -o json > "$OUT/frontend-service.json"
fi
printf 'build_id=%s\nnamespace=%s\ncredentials=%s\n' "$BUILD_ID" "$NAMESPACE" "$OUT/bootstrap-admin-password.txt"

View File

@ -51,7 +51,14 @@ def main() -> int:
# the runtime copy so pick-place validation measures real bilateral contact.
runtime_xml = runtime_xml.replace(
'contype="0" conaffinity="0" density="0" group="2"/>',
'contype="1" conaffinity="1" density="0" friction="1.2 0.01 0.001" group="3"/>',
'contype="1" conaffinity="1" density="0" friction="2.0 0.02 0.002" group="3"/>',
)
runtime_xml = runtime_xml.replace(
'name="left_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.003"',
'name="left_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.010"',
).replace(
'name="right_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.003"',
'name="right_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.010"',
)
runtime_rm65.write_text(runtime_xml, encoding="utf-8")

View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}"
K=(kubectl --kubeconfig "$KUBECONFIG_PATH" -n real2sim-poc)
for deployment in real2sim-frontend real2sim-api real2sim-worker; do
"${K[@]}" rollout undo "deployment/$deployment"
"${K[@]}" rollout status "deployment/$deployment" --timeout=3600s
done
echo "rollback completed; PV and BOS objects were preserved"

View File

@ -34,12 +34,6 @@ def main() -> int:
parser.add_argument("--width", type=int, default=960)
parser.add_argument("--height", type=int, default=540)
parser.add_argument("--no-render", action="store_true")
parser.add_argument(
"--grasp-mode",
choices=("physical", "kinematic_attachment"),
default="physical",
help="Use contact physics by default; kinematic attachment is retained only for diagnostics.",
)
args = parser.parse_args()
config = load_config()
@ -61,7 +55,6 @@ def main() -> int:
width=args.width,
height=args.height,
render=not args.no_render,
grasp_mode=args.grasp_mode,
)
artifacts = {}
@ -81,7 +74,6 @@ def run_rollout(
width: int,
height: int,
render: bool,
grasp_mode: str = "physical",
) -> dict[str, Any]:
import mujoco
@ -105,7 +97,7 @@ def run_rollout(
qpos_adrs = [int(model.jnt_qposadr[jid]) for jid in joint_ids]
dof_adrs = [int(model.jnt_dofadr[jid]) for jid in joint_ids]
duration_scale = 1 if grasp_mode == "kinematic_attachment" else 8
duration_scale = 8
phases = [
("home", np.array([0.42, 0.00, 0.72]), 0.0, 35 * duration_scale, False),
("approach", np.array([0.71, 0.015, 0.42]), 0.0, 55 * duration_scale, False),
@ -123,7 +115,6 @@ def run_rollout(
if render:
renderer = mujoco.Renderer(model, height=height, width=width)
attached = False
contact_steps = 0
bilateral_contact_steps = 0
phase_contacts = {phase: {"contact_steps": 0, "bilateral_contact_steps": 0} for phase, *_ in phases}
@ -135,7 +126,7 @@ def run_rollout(
started = time.time()
frame_stride = max(1, round((1.0 / model.opt.timestep) / fps))
step_index = 0
for phase, target, gripper_ctrl, count, attach_cube in phases:
for phase, target, gripper_ctrl, count, _contact_expected in phases:
if phase == "grasp":
settled_cube_z = float(data.qpos[cube_qadr + 2])
max_cube_z = settled_cube_z
@ -145,14 +136,7 @@ def run_rollout(
current = data.ctrl[:6].copy()
data.ctrl[:6] = current * (1 - alpha) + q_target * alpha
data.ctrl[6] = gripper_ctrl
if grasp_mode == "kinematic_attachment" and (attach_cube or attached):
attached = attach_cube
if attached:
set_cube_pose(data, cube_qadr, data.site_xpos[site_id] + np.array([0.0, 0.0, -0.095]))
mujoco.mj_step(model, data)
if grasp_mode == "kinematic_attachment" and attached:
set_cube_pose(data, cube_qadr, data.site_xpos[site_id] + np.array([0.0, 0.0, -0.095]))
mujoco.mj_forward(model, data)
touching_fingers = cube_finger_contacts(data, cube_geom, finger_geoms)
accumulate_contact_metrics(model, data, cube_geom, finger_geoms, contact_force_metrics)
if touching_fingers:
@ -193,7 +177,7 @@ def run_rollout(
place_error = float(np.linalg.norm(final_cube - target_cube))
physical_success = bilateral_contact_steps > 0 and lift_height >= 0.10 and place_error <= 0.12
summary = {
"mode": "scripted_ik_physical_contact" if grasp_mode == "physical" else "scripted_ik_with_cube_attachment",
"mode": "scripted_ik_physical_contact",
"steps": step_index,
"samples": len(trajectory),
"sim_time": float(data.time),
@ -210,22 +194,18 @@ def run_rollout(
"phase_contacts": phase_contacts,
"gripper_qpos_range": gripper_qpos_range,
"contact_force_metrics": contact_force_metrics,
"physical_success": physical_success if grasp_mode == "physical" else None,
"physical_success": physical_success,
"elapsed_ms": round((time.time() - started) * 1000),
}
report = {
"schema": "real2sim.rm65_pick_place_rollout_report.v1",
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"ok": physical_success if grasp_mode == "physical" else True,
"ok": physical_success,
"runtime": "mujoco",
"scene": str(sim_path),
"summary": summary,
"phases": [phase for phase, *_ in phases],
"note": (
"Scripted IK with contact-physics grasp; the cube pose is never overwritten. This is not a learned policy."
if grasp_mode == "physical"
else "Diagnostic-only rollout. Cube follows the gripper during grasp/lift/place; this is not a learned policy."
),
"note": "Scripted IK with contact-physics grasp; the cube pose is never overwritten. This is not a learned policy.",
}
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
artifact_paths["report"] = str(report_path)
@ -284,12 +264,6 @@ def solve_ik(model: Any, data: Any, site_id: int, target: np.ndarray, qpos_adrs:
return q
def set_cube_pose(data: Any, qadr: int, pos: np.ndarray) -> None:
data.qpos[qadr : qadr + 3] = pos
data.qpos[qadr + 3 : qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
data.qvel[qadr : qadr + 6] = 0
def sample_state(model: Any, data: Any, phase: str, step: int, site_id: int, cube_qadr: int) -> dict[str, Any]:
return {
"step": step,

View File

@ -0,0 +1,195 @@
from __future__ import annotations
import argparse
import importlib.util
import json
import pickle
import sys
import time
from pathlib import Path
from typing import Any
import numpy as np
from .rm65_controller import RM65PickPlaceRuntime, _write_preview_video
def load_policy(model_dir: Path, act_root: Path, max_timesteps: int):
import torch
sys.path.insert(0, str(act_root))
from configurations.task_configs.config_tools.basic_configer import config_policy
config_path = model_dir / "task_config.py"
checkpoint = model_dir / "core" / "policy_best.ckpt"
stats_path = model_dir / "core" / "dataset_stats.pkl"
for path in (config_path, checkpoint, stats_path):
if not path.is_file():
raise FileNotFoundError(path)
spec = importlib.util.spec_from_file_location("real2sim_act_task_config", config_path)
if spec is None or spec.loader is None:
raise RuntimeError("unable to load ACT task config")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
task_config = module.TASK_CONFIG
merged = {**task_config["common"], **task_config["train"]}
merged["learning_rate"] = task_config["train"]["learning_rate"]
policy_config = config_policy(merged)
policy_config["ckpt_path"] = str(checkpoint)
policy_config["max_timesteps"] = max_timesteps
policy = module.policy_maker(policy_config, "eval")
with stats_path.open("rb") as handle:
stats = pickle.load(handle)
if not all(key in stats for key in ("qpos_mean", "qpos_std", "action_mean", "action_std")):
raise ValueError("ACT dataset_stats.pkl is missing normalization arrays")
return policy, stats, task_config["common"]["camera_names"], torch
def infer(
*,
scene_path: Path,
model_dir: Path,
act_root: Path,
out_dir: Path,
episodes: int,
steps: int,
seed: int,
minimum_success_rate: float,
width: int,
height: int,
) -> dict[str, Any]:
out_dir.mkdir(parents=True, exist_ok=True)
policy, stats, camera_names, torch = load_policy(model_dir, act_root, steps)
if camera_names != ["overview"]:
raise ValueError(f"RM65 runtime currently exposes camera ['overview']; model expects {camera_names}")
runtime = RM65PickPlaceRuntime(scene_path, width=width, height=height, seed=seed)
results: list[dict[str, Any]] = []
all_actions: list[np.ndarray] = []
try:
for episode in range(episodes):
policy.reset()
cube, target, settled_z = _reset_randomized(runtime)
trajectory: list[dict[str, Any]] = []
frames: list[np.ndarray] = []
max_cube_z = settled_z
bilateral_steps = 0
for step in range(steps):
runtime.renderer.update_scene(runtime.data, camera="overview")
frame = runtime.renderer.render().copy()
qpos = np.concatenate([runtime.data.qpos[runtime.qpos_adrs], [runtime.data.qpos[runtime.gripper_qadr]]]).astype(np.float32)
normalized_qpos = (qpos - np.asarray(stats["qpos_mean"], dtype=np.float32)) / np.asarray(stats["qpos_std"], dtype=np.float32)
image = torch.from_numpy(frame).permute(2, 0, 1).unsqueeze(0).float().cuda() / 255.0
qpos_tensor = torch.from_numpy(normalized_qpos).unsqueeze(0).float().cuda()
with torch.inference_mode():
predicted = policy(qpos_tensor, image)
normalized_action = predicted[0, 0].detach().cpu().numpy()
action = normalized_action * np.asarray(stats["action_std"], dtype=np.float32) + np.asarray(stats["action_mean"], dtype=np.float32)
action = np.asarray(action, dtype=np.float32)
if action.shape != (7,):
raise ValueError(f"ACT action shape must be (7,), received {action.shape}")
for index in range(7):
action[index] = np.clip(action[index], runtime.ctrl_ranges[index, 0], runtime.ctrl_ranges[index, 1])
runtime.data.ctrl[:7] = action
for _ in range(10):
runtime.mujoco.mj_step(runtime.model, runtime.data)
touching = runtime.finger_contacts()
bilateral_steps += int(len(touching) == 2)
max_cube_z = max(max_cube_z, float(runtime.data.qpos[runtime.cube_qadr + 2]))
all_actions.append(action.copy())
frames.append(frame)
trajectory.append(
{
"step": step,
"time": float(runtime.data.time),
"qpos": qpos.tolist(),
"action": action.tolist(),
"cube_pos": runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3].astype(float).tolist(),
"target": target.tolist(),
"bilateral_contact": len(touching) == 2,
}
)
final_cube = runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3].copy()
lift_height = float(max_cube_z - settled_z)
place_error = float(np.linalg.norm(final_cube - target))
success = bilateral_steps > 0 and lift_height >= 0.10 and place_error <= 0.05
result = {
"episode": episode,
"success": success,
"lift_height": lift_height,
"place_error": place_error,
"bilateral_contact_steps": bilateral_steps,
"initial_cube": cube.tolist(),
"target": target.tolist(),
}
results.append(result)
(out_dir / f"episode_{episode:03d}_trajectory.json").write_text(json.dumps({"schema": "real2sim.act_rollout.v1", "result": result, "samples": trajectory}, ensure_ascii=False) + "\n", encoding="utf-8")
_write_preview_video(out_dir / f"episode_{episode:03d}.mp4", frames)
finally:
runtime.close()
success_rate = sum(int(item["success"]) for item in results) / max(1, len(results))
action_matrix = np.asarray(all_actions, dtype=np.float32)
action_std = action_matrix.std(axis=0).tolist() if len(action_matrix) else []
nonconstant = bool(len(action_matrix) > 1 and np.max(np.std(action_matrix, axis=0)) > 1e-5)
report = {
"schema": "real2sim.discoverse_act_inference.v1",
"runtime": "mujoco",
"scene": str(scene_path),
"model": str(model_dir),
"episodes": results,
"success_rate": success_rate,
"minimum_success_rate": minimum_success_rate,
"action_std": action_std,
"actions_nonconstant": nonconstant,
"pass": success_rate >= minimum_success_rate and nonconstant,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
(out_dir / "inference_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return report
def _reset_randomized(runtime: RM65PickPlaceRuntime) -> tuple[np.ndarray, np.ndarray, float]:
runtime.mujoco.mj_resetDataKeyframe(runtime.model, runtime.data, 0)
cube = np.array([runtime.rng.uniform(0.645, 0.655), runtime.rng.uniform(-0.010, 0.010), 0.33], dtype=float)
target = np.array([runtime.rng.uniform(0.42, 0.51), runtime.rng.uniform(0.13, 0.22), 0.31], dtype=float)
runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3] = cube
runtime.data.qpos[runtime.cube_qadr + 3 : runtime.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
cube_dof = int(runtime.model.jnt_dofadr[runtime.cube_joint])
runtime.data.qvel[cube_dof : cube_dof + 6] = 0
runtime.model.body_pos[runtime.target_body] = np.array([target[0], target[1], 0.335])
runtime.mujoco.mj_forward(runtime.model, runtime.data)
for _ in range(120):
runtime.mujoco.mj_step(runtime.model, runtime.data)
return cube, target, float(runtime.data.qpos[runtime.cube_qadr + 2])
def main() -> int:
parser = argparse.ArgumentParser(description="Run DISCOVERSE ACT in the physical RM65 MuJoCo scene")
parser.add_argument("--scene", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--act-root", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--episodes", type=int, default=10)
parser.add_argument("--steps", type=int, default=200)
parser.add_argument("--seed", type=int, default=11)
parser.add_argument("--minimum-success-rate", type=float, default=0.5)
parser.add_argument("--width", type=int, default=320)
parser.add_argument("--height", type=int, default=240)
args = parser.parse_args()
report = infer(
scene_path=args.scene,
model_dir=args.model,
act_root=args.act_root,
out_dir=args.out,
episodes=args.episodes,
steps=args.steps,
seed=args.seed,
minimum_success_rate=args.minimum_success_rate,
width=args.width,
height=args.height,
)
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["pass"] else 2
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -1,6 +1,8 @@
from __future__ import annotations
import json
import hashlib
from functools import lru_cache
import math
import os
import shutil
@ -15,6 +17,7 @@ from urllib.parse import quote, unquote, urlparse
from urllib.request import urlopen
from .config import PlatformConfig
from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path
from .store import PlatformStore
@ -61,16 +64,26 @@ def scan_dataset(path: Path) -> dict[str, Any]:
def run_doctor(config: PlatformConfig) -> dict[str, Any]:
core_checks = [
"workspace_writable",
"expert_policy",
"training",
"inference",
"media_prepare_deps",
"ffmpeg",
"spark_bundle",
"three_bundle",
]
core_checks = (
[
"workspace_writable",
"rm65_controller",
"discoverse_act",
"torch_cuda",
"act_import",
"media_prepare_deps",
"ffmpeg",
"colmap",
"gs_video_reconstruct",
"anysplat_pinned",
"mujoco_python",
"motrixsim_python",
"nvidia_smi",
]
if config.production_mode
else ["workspace_writable", "expert_policy", "training", "inference", "media_prepare_deps", "ffmpeg", "spark_bundle", "three_bundle"]
)
act_root = config.discoverse_root / "policies" / "act"
checks = {
"workspace_writable": _is_writable(config.workspace_dir),
"expert_policy": (config.expert_policy_root / "src/real2sim_expert/collect.py").is_file(),
@ -91,9 +104,15 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]:
"mujoco_python": _module_available("mujoco"),
"service_process": os.getpid() > 0,
"nvidia_smi": shutil.which("nvidia-smi") is not None,
"rm65_controller": Path(__file__).with_name("rm65_controller.py").is_file(),
"discoverse_act": _git_head(config.discoverse_root) == config.discoverse_commit and _git_head(act_root) == config.act_commit,
"torch_cuda": _torch_cuda_ready(),
"act_import": _act_runtime_import_ready(str(config.discoverse_root), config.discoverse_commit, config.act_commit),
"anysplat_pinned": _git_head(config.anysplat_root) == config.anysplat_commit and config.anysplat_python.exists(),
"motrixsim_python": _module_available("motrixsim"),
}
return {
"ok": all(checks[name] for name in core_checks),
"ok": all(bool(checks[name]) for name in core_checks),
"core_checks": core_checks,
"warnings": [
name
@ -108,6 +127,8 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]:
"cuda_home",
"colmap",
"nvidia_smi",
"torch_cuda",
"act_import",
"three_bundle",
]
if not checks.get(name)
@ -148,6 +169,8 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A
return run_reconstruction_preflight(config, store, task, emit)
if task_type == "hf_import":
return run_hf_import(config, store, task, emit)
if task_type == "bos_import":
return run_bos_import(config, store, task, emit)
if task_type == "simulator_validation":
return run_simulator_validation(config, store, task, emit)
if task_type == "expert_collection":
@ -165,6 +188,42 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A
raise ValueError(f"unknown task type: {task_type}")
def sync_task_outputs(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]:
storage = BosStorage(config)
if not storage.configured:
if config.production_mode:
raise RuntimeError("生产任务产物无法持久化:BOS 尚未配置")
return result
_task_progress(store, task, 96, "持久化产物", "正在将任务报告和业务产物上传到 BOS")
run_dir = Path(task["run_dir"])
if run_dir.is_dir():
_upload_directory(storage, run_dir, area="runs", relative_root=task["id"])
resource_specs = [
("dataset_id", "datasets", store.get_dataset),
("scene_id", "scenes", store.get_scene),
("model_id", "models", store.get_model),
]
persisted: dict[str, Any] = {}
for result_key, area, getter in resource_specs:
entity_id = str(result.get(result_key) or "")
if not entity_id:
continue
entity = getter(entity_id)
path = Path(str(entity["path"]))
root = path if path.is_dir() else path.parent
if not root.is_dir():
raise FileNotFoundError(f"resource output directory not found: {root}")
_upload_directory(storage, root, area=area, relative_root=entity_id)
checksum = _directory_digest(root)
uri = storage.uri_for_key(storage.object_key(area, f"{entity_id}/"))
store.update_entity_storage(area, entity_id, storage_uri=uri, checksum=checksum, cache_state="local")
persisted[result_key] = {"storage_uri": uri, "checksum": checksum}
result = dict(result)
result["persistent_outputs"] = persisted
result["run_storage_uri"] = storage.uri_for_key(storage.object_key("runs", f"{task['id']}/"))
return result
def _task_progress(store: PlatformStore, task: dict[str, Any] | str, progress: int, stage: str, message: str = "") -> None:
task_id = task if isinstance(task, str) else task["id"]
store.update_task(task_id, progress=progress, stage=stage, message=message)
@ -504,6 +563,66 @@ def run_hf_import(config: PlatformConfig, store: PlatformStore, task: dict[str,
return {"dataset_id": dataset["id"], "dataset_path": str(target), "manifest_path": str(manifest_path), "scan": scan}
def run_bos_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
params = task["params"]
storage = BosStorage(config)
raw_prefix = normalize_relative_path(str(params.get("prefix") or ""))
prefix_parts = raw_prefix.split("/", 1)
area = prefix_parts[0] if prefix_parts[0] in ALLOWED_AREAS else "datasets"
relative_prefix = prefix_parts[1] if prefix_parts[0] in ALLOWED_AREAS and len(prefix_parts) > 1 else raw_prefix
_task_progress(store, task, 18, "读取 BOS", "正在列举授权前缀内的对象")
objects = storage.list_objects(area=area, relative_prefix=relative_prefix, max_keys=int(params.get("max_objects", 1000)))
objects = [item for item in objects if not item["key"].endswith("/")]
if not objects:
raise ValueError("BOS 前缀中没有可导入对象")
dataset_id = uuid.uuid4().hex
out_dir = config.datasets_dir / dataset_id
out_dir.mkdir(parents=True, exist_ok=False)
common_prefix = storage.object_key(area, relative_prefix)
_task_progress(store, task, 30, "下载数据", f"正在下载 {len(objects)} 个真实 BOS 对象")
for index, item in enumerate(objects, start=1):
relative = item["key"][len(common_prefix):].lstrip("/") or Path(item["key"]).name
storage.download_file(item["storage_uri"], out_dir / relative)
if index % max(1, len(objects) // 10) == 0:
_task_progress(store, task, 30 + int(45 * index / len(objects)), "下载数据", f"已下载 {index}/{len(objects)} 个对象")
scan = scan_dataset(out_dir)
digest = _directory_digest(out_dir)
source_uri = f"bos://{storage.bucket}/{common_prefix}"
manifest = {
"schema": "real2sim.dataset_manifest.v2",
"id": dataset_id,
"name": params.get("name") or Path(relative_prefix.rstrip("/")).name,
"source_type": "bos_prefix",
"storage_uri": source_uri,
"checksum": digest,
"version": digest[:12],
"object_count": len(objects),
"objects": objects,
"scan": scan,
}
manifest_path = out_dir / "dataset_manifest.json"
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
dataset = store.create_dataset(
item_id=dataset_id,
name=str(manifest["name"]),
path=out_dir,
scan=scan,
source_type="bos_prefix",
source_uri=source_uri,
version=digest[:12],
parent_task_id=task["id"],
manifest_path=manifest_path,
metrics={"object_count": len(objects), "size_bytes": sum(int(item["size_bytes"]) for item in objects)},
tags=["bos", "imported"],
storage_uri=source_uri,
checksum=digest,
cache_state="local",
)
_index_artifacts(store, task["id"], out_dir)
_task_progress(store, task, 95, "登记数据集", "BOS 数据及校验清单已登记")
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "storage_uri": source_uri, "object_count": len(objects), "checksum": digest}
def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
params = task["params"]
run_dir = Path(task["run_dir"])
@ -564,8 +683,50 @@ def run_expert_collection(config: PlatformConfig, store: PlatformStore, task: di
run_dir = Path(task["run_dir"])
scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json"
out_dir = run_dir / "dataset"
policy_preset = str(params.get("policy_preset") or "pick_and_place_v1")
if policy_preset != "pick_and_place_v1":
policy_preset = str(params.get("policy_preset") or ("rm65_pick_place_6d_ik" if config.production_mode else "pick_and_place_v1"))
if policy_preset == "rm65_pick_place_6d_ik":
scene_path = _resolve_rm65_runtime_scene(store, params, scene_path)
required_successes = int(params.get("episodes", 50))
max_attempts = int(params.get("max_attempts", max(required_successes * 3, required_successes)))
command = [
sys.executable,
"-m",
"real2sim_platform.rm65_controller",
"--scene",
str(scene_path),
"--out",
str(out_dir),
"--required-successes",
str(required_successes),
"--max-attempts",
str(max_attempts),
"--steps",
str(int(params.get("steps", 200))),
"--seed",
str(int(params.get("seed", 7))),
"--width",
str(int(params.get("image_width", 320))),
"--height",
str(int(params.get("image_height", 240))),
]
_task_progress(store, task, 28, "物理采集", "RM65 正在通过 6D IK、双指接触和真实物理步进采集成功 episode")
run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src"]), store=store, task_id=task["id"])
manifest = _read_json(out_dir / "manifest.json")
dataset = store.create_dataset(
name=params.get("name", f"RM65 ACT dataset {task['id'][:8]}"),
path=out_dir,
scan={"input_kind": "discoverse_act_hdf5", "samples": manifest.get("successful_episodes"), "manifest": str(out_dir / "manifest.json")},
source_type="expert_collection",
source_uri=str(scene_path),
parent_task_id=task["id"],
manifest_path=out_dir / "manifest.json",
metrics={"samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate"), "policy_preset": policy_preset},
tags=["discoverse_act_hdf5", "rm65", "physical_contact", policy_preset],
)
_index_artifacts(store, task["id"], out_dir)
_task_progress(store, task, 95, "登记数据集", "真实采集 HDF5、接触记录和成功报告已登记")
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate")}
if policy_preset != "pick_and_place_v1" or config.production_mode:
raise ValueError(f"unsupported expert policy preset: {policy_preset}")
_task_progress(store, task, 18, "准备采集", "正在加载场景、专家策略和机器人配置")
command = [
@ -615,7 +776,9 @@ def run_training(config: PlatformConfig, store: PlatformStore, task: dict[str, A
_task_progress(store, task, 18, "准备训练", "正在加载训练数据集和训练配方")
dataset_path = _dataset_path(store, params)
out_dir = run_dir / "model"
recipe = str(params.get("recipe") or "bc_v2")
recipe = str(params.get("recipe") or ("discoverse_act" if config.production_mode else "bc_v2"))
if recipe == "discoverse_act":
return _run_discoverse_act_training(config, store, task, dataset_path, out_dir)
recipe_config = {
"bc_v2": {"regularization": 1e-3, "base_model": "facebook/dinov2-small"},
"bc_fast": {"regularization": 1e-2, "base_model": "facebook/dinov2-small"},
@ -673,6 +836,44 @@ def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str,
scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json"
model_path = _model_path(store, params)
out_dir = run_dir / "inference"
model = store.get_model(str(params["model_id"])) if params.get("model_id") else None
model_recipe = str(((model or {}).get("metadata") or {}).get("training", {}).get("recipe") or params.get("recipe") or "")
if model_recipe == "discoverse_act":
scene_path = _resolve_rm65_runtime_scene(store, params, scene_path)
act_root = config.discoverse_root / "policies" / "act"
if _git_head(act_root) != config.act_commit:
raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}")
command = [
sys.executable,
"-m",
"real2sim_platform.act_inference",
"--scene",
str(scene_path),
"--model",
str(model_path),
"--act-root",
str(act_root),
"--out",
str(out_dir),
"--episodes",
str(int(params.get("episodes", 10))),
"--steps",
str(int(params.get("steps", 200))),
"--seed",
str(int(params.get("seed", 11))),
"--minimum-success-rate",
str(float(params.get("minimum_success_rate", 0.5))),
]
_task_progress(store, task, 38, "ACT 在线推理", "DISCOVERSE ACT 正在真实 MuJoCo 场景中闭环控制 RM65")
run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src", act_root, config.discoverse_root]), store=store, task_id=task["id"])
report = _read_json(out_dir / "inference_report.json")
if not report.get("actions_nonconstant"):
raise RuntimeError("ACT 推理动作是常量,拒绝登记为有效推理")
_index_artifacts(store, task["id"], out_dir)
_task_progress(store, task, 95, "索引推理产物", "真实 rollout 视频、轨迹和成功率报告已登记")
return {"episodes": len(report.get("episodes", [])), "success_rate": report.get("success_rate"), "report": str(out_dir / "inference_report.json")}
if config.production_mode:
raise ValueError("生产模式只允许 DISCOVERSE ACT 模型在线推理")
command = [
sys.executable,
"-m",
@ -695,6 +896,97 @@ def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str,
return {"rollout_steps": len(output.get("rollout", [])), "viewer": str(out_dir / "viewer.html")}
def _run_discoverse_act_training(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], dataset_path: Path, out_dir: Path) -> dict[str, Any]:
params = task["params"]
run_dir = Path(task["run_dir"])
act_root = config.discoverse_root / "policies" / "act"
if _git_head(act_root) != config.act_commit:
raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}")
dataset_info = _inspect_act_dataset(dataset_path)
if dataset_info["episode_count"] < 2:
raise ValueError("DISCOVERSE ACT 至少需要 2 个真实 HDF5 episode")
out_dir.mkdir(parents=True, exist_ok=True)
config_root = run_dir / "generated_config"
task_name = f"real2sim_{task['id'][:12]}"
config_path = config_root / "configurations" / "task_configs" / f"{task_name}.py"
config_path.parent.mkdir(parents=True, exist_ok=True)
epochs = int(params.get("epochs", 100))
validate_every = max(1, min(int(params.get("validate_every", max(1, epochs // 5))), epochs))
save_every = max(1, min(int(params.get("save_every", validate_every)), epochs))
batch_size = max(1, min(int(params.get("batch_size", 8)), dataset_info["episode_count"] - 1))
chunk_size = max(2, min(int(params.get("chunk_size", 32)), dataset_info["episode_length"]))
core_dir = out_dir / "core"
checkpoint_dir = out_dir / "checkpoints"
config_path.write_text(
_act_task_config_source(
task_name=task_name,
dataset_path=dataset_path,
checkpoint_dir=checkpoint_dir,
stats_path=core_dir / "dataset_stats.pkl",
camera_names=dataset_info["camera_names"],
state_dim=dataset_info["state_dim"],
action_dim=dataset_info["action_dim"],
epochs=epochs,
batch_size=batch_size,
chunk_size=chunk_size,
validate_every=validate_every,
save_every=save_every,
),
encoding="utf-8",
)
command = [sys.executable, "policy_train.py", "-tn", task_name, "-ne", str(epochs), "-ve", str(validate_every), "-se", str(save_every), "--seed", str(int(params.get("seed", 1)))]
env = _env_with_paths(config, [config_root, act_root, config.discoverse_root])
env["MPLBACKEND"] = "Agg"
_task_progress(store, task, 34, "ACT 训练", f"正在使用 {dataset_info['episode_count']} 个真实 episode 训练 DISCOVERSE ACT")
run_logged(command, cwd=act_root, log_path=run_dir / "logs" / "task.log", env=env, store=store, task_id=task["id"])
checkpoint = core_dir / "policy_best.ckpt"
stats_path = core_dir / "dataset_stats.pkl"
if not checkpoint.is_file() or checkpoint.stat().st_size == 0 or not stats_path.is_file():
raise RuntimeError("DISCOVERSE ACT 训练结束但未生成非空 policy_best.ckpt 或 dataset_stats.pkl")
description = (core_dir / "description.txt").read_text(encoding="utf-8").strip() if (core_dir / "description.txt").is_file() else ""
best_loss = _parse_best_loss(description)
if best_loss is None or not math.isfinite(best_loss):
raise RuntimeError("ACT 训练指标不是有限数值")
model_card = {
"schema": "real2sim.model_card.v2",
"name": params.get("name", f"RM65 ACT {task['id'][:8]}"),
"framework": "DISCOVERSE ACT",
"training": {
"recipe": "discoverse_act",
"discoverse_commit": config.discoverse_commit,
"act_commit": config.act_commit,
"dataset_path": str(dataset_path),
"epochs": epochs,
"batch_size": batch_size,
"chunk_size": chunk_size,
"camera_names": dataset_info["camera_names"],
"state_dim": dataset_info["state_dim"],
"action_dim": dataset_info["action_dim"],
"config_path": str(config_path),
},
"metrics": {"best_validation_loss": best_loss},
"artifacts": {"checkpoint": str(checkpoint), "dataset_stats": str(stats_path)},
"description": description,
}
card_path = out_dir / "model_card.json"
card_path.write_text(json.dumps(model_card, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
shutil.copy2(config_path, out_dir / "task_config.py")
model = store.create_model(
name=str(model_card["name"]),
path=out_dir,
metadata=model_card,
source_type="training",
source_uri=str(dataset_path),
parent_task_id=task["id"],
manifest_path=card_path,
metrics=model_card["metrics"],
tags=["policy", "discoverse_act", "rm65"],
)
_index_artifacts(store, task["id"], out_dir)
_task_progress(store, task, 95, "索引 ACT 模型", "checkpoint、损失曲线、模型卡和数据血缘已登记")
return {"model_id": model["id"], "model_path": str(out_dir), "metrics": model_card["metrics"], "checkpoint": str(checkpoint)}
def run_renderer_benchmark(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
run_dir = Path(task["run_dir"])
run_dir.mkdir(parents=True, exist_ok=True)
@ -1550,6 +1842,35 @@ def _scene_path(store: PlatformStore, params: dict[str, Any]) -> Path | None:
return None
def _resolve_rm65_runtime_scene(store: PlatformStore, params: dict[str, Any], fallback: Path) -> Path:
candidates: list[Path] = []
if fallback.suffix.lower() == ".xml":
candidates.append(fallback)
if params.get("scene_id"):
scene = store.get_scene(str(params["scene_id"]))
metadata = scene.get("metadata") or {}
package = Path(str(metadata.get("package") or ""))
manifest_path = Path(str(scene.get("manifest_path") or ""))
manifest = _read_optional_json(manifest_path)
for asset in manifest.get("assets", []) if isinstance(manifest.get("assets"), list) else []:
if not isinstance(asset, dict):
continue
name = str(asset.get("name") or asset.get("path") or "").lower()
if "rm65" in name and name.endswith(".xml"):
candidates.append(Path(str(asset.get("path") or "")))
if str(package):
candidates.extend([
package / "simulator_assets" / "rm65_table_pick_place.xml",
package / "scene_mujoco.xml",
package / "mjcf" / "scene.xml",
])
candidates.extend([fallback.parent / "simulator_assets" / "rm65_table_pick_place.xml", fallback.parent / "scene_mujoco.xml"])
for candidate in candidates:
if candidate.is_file():
return candidate
raise FileNotFoundError("所选场景没有可执行的 RM65 pick-place MJCF;请先在场景管理中绑定 RM65 控制器和碰撞资产")
def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path:
if params.get("model_path"):
return Path(str(params["model_path"]))
@ -1558,6 +1879,147 @@ def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path:
raise ValueError("inference requires model_id or model_path")
def _directory_digest(root: Path) -> str:
digest = hashlib.sha256()
for path in sorted(item for item in root.rglob("*") if item.is_file()):
relative = path.relative_to(root).as_posix()
digest.update(relative.encode("utf-8"))
with path.open("rb") as handle:
while chunk := handle.read(8 * 1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _upload_directory(storage: BosStorage, root: Path, *, area: str, relative_root: str) -> None:
for path in sorted(item for item in root.rglob("*") if item.is_file()):
storage.upload_file(
path,
area=area,
relative_path=f"{relative_root}/{path.relative_to(root).as_posix()}",
metadata={"sha256": _file_sha256(path)},
)
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
while chunk := handle.read(8 * 1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _git_head(root: Path) -> str:
try:
return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=5).strip()
except Exception:
return ""
def _inspect_act_dataset(dataset_path: Path) -> dict[str, Any]:
try:
import h5py
except ImportError as exc:
raise RuntimeError("DISCOVERSE ACT training requires h5py") from exc
episodes = sorted(dataset_path.glob("episode_*.hdf5"))
if not episodes:
raise ValueError("训练数据目录中没有 episode_*.hdf5")
lengths: list[int] = []
cameras: list[str] = []
state_dim = 0
action_dim = 0
for path in episodes:
with h5py.File(path, "r") as root:
if "/observations/qpos" not in root or "/action" not in root or "/observations/images" not in root:
raise ValueError(f"ACT HDF5 缺少 qpos/action/images: {path.name}")
current_cameras = sorted(str(name) for name in root["/observations/images"].keys())
current_state_dim = int(root["/observations/qpos"].shape[1])
current_action_dim = int(root["/action"].shape[1])
if not cameras:
cameras = current_cameras
state_dim = current_state_dim
action_dim = current_action_dim
if current_cameras != cameras or current_state_dim != state_dim or current_action_dim != action_dim:
raise ValueError("ACT 数据集的相机、状态维度或动作维度不一致")
length = int(root["/action"].shape[0])
if length < 2:
raise ValueError(f"ACT episode 过短: {path.name}")
lengths.append(length)
return {
"episode_count": len(episodes),
"episode_length": min(lengths),
"camera_names": cameras,
"state_dim": state_dim,
"action_dim": action_dim,
}
def _act_task_config_source(
*,
task_name: str,
dataset_path: Path,
checkpoint_dir: Path,
stats_path: Path,
camera_names: list[str],
state_dim: int,
action_dim: int,
epochs: int,
batch_size: int,
chunk_size: int,
validate_every: int,
save_every: int,
) -> str:
return f'''from configurations.task_configs.template import TASK_CONFIG_DEFAULT, activator, set_paths
def policy_maker(config: dict, stage=None):
from policies.act.act import ACTPolicy
from policies.common.maker import post_init_policies
policy = ACTPolicy(config)
post_init_policies([policy], stage, [config["ckpt_path"]])
return policy
@activator(False)
def augment_images(image):
return image
set_paths({str(dataset_path)!r}, {str(checkpoint_dir)!r}, {str(stats_path)!r})
TASK_CONFIG_DEFAULT["common"]["camera_names"] = {camera_names!r}
TASK_CONFIG_DEFAULT["common"]["state_dim"] = {state_dim}
TASK_CONFIG_DEFAULT["common"]["action_dim"] = {action_dim}
TASK_CONFIG_DEFAULT["common"]["policy_config"]["policy_maker"] = policy_maker
TASK_CONFIG_DEFAULT["common"]["policy_config"]["temporal_agg"] = True
TASK_CONFIG_DEFAULT["common"]["policy_config"]["chunk_size"] = {chunk_size}
TASK_CONFIG_DEFAULT["common"]["policy_config"]["num_queries"] = {chunk_size}
TASK_CONFIG_DEFAULT["train"]["load_data"]["data_type"] = "hdf5"
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_episodes"] = "ALL"
TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_train"] = {batch_size}
TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_validate"] = {batch_size}
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_train"] = 2
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_validate"] = 2
TASK_CONFIG_DEFAULT["train"]["load_data"]["observation_slice"] = None
TASK_CONFIG_DEFAULT["train"]["load_data"]["action_slice"] = None
TASK_CONFIG_DEFAULT["train"]["num_epochs"] = {epochs}
TASK_CONFIG_DEFAULT["train"]["validate_every"] = {validate_every}
TASK_CONFIG_DEFAULT["train"]["save_every"] = {save_every}
TASK_CONFIG_DEFAULT["train"]["eval_every"] = 0
TASK_CONFIG = TASK_CONFIG_DEFAULT
'''
def _parse_best_loss(description: str) -> float | None:
import re
match = re.search(r"val loss\s+([0-9.eE+-]+)", description)
if not match:
return None
try:
return float(match.group(1))
except ValueError:
return None
def _write_dry_run_scene(package_dir: Path, input_path: Path, renderer: str, simulators: list[str]) -> Path:
package_dir.mkdir(parents=True, exist_ok=True)
scene = {
@ -2465,6 +2927,40 @@ def _module_available(name: str) -> bool:
return False
@lru_cache(maxsize=1)
def _torch_cuda_ready() -> bool:
if not _module_available("torch"):
return False
completed = subprocess.run(
[sys.executable, "-c", "import torch; raise SystemExit(0 if torch.cuda.is_available() else 2)"],
text=True,
capture_output=True,
timeout=30,
check=False,
)
return completed.returncode == 0
@lru_cache(maxsize=4)
def _act_runtime_import_ready(discoverse_root_value: str, discoverse_commit: str, act_commit: str) -> bool:
discoverse_root = Path(discoverse_root_value)
act_root = discoverse_root / "policies" / "act"
if _git_head(discoverse_root) != discoverse_commit or _git_head(act_root) != act_commit or not _torch_cuda_ready():
return False
env = os.environ.copy()
env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep)
completed = subprocess.run(
[sys.executable, "-c", "from policy import ACTPolicy; assert ACTPolicy is not None"],
cwd=act_root,
env=env,
text=True,
capture_output=True,
timeout=45,
check=False,
)
return completed.returncode == 0
def _is_writable(path: Path) -> bool:
try:
path.mkdir(parents=True, exist_ok=True)

View File

@ -0,0 +1,143 @@
from __future__ import annotations
import hashlib
import re
import secrets
import time
from pathlib import Path
from typing import Any
from .config import PlatformConfig
from .store import PlatformStore
SESSION_COOKIE = "real2sim_session"
VALID_ROLES = {"admin", "engineer", "viewer"}
_USERNAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$")
class AuthenticationError(ValueError):
pass
class AuthManager:
def __init__(self, store: PlatformStore, config: PlatformConfig) -> None:
self.store = store
self.config = config
self._hasher = _password_hasher() if config.auth_enabled else None
def bootstrap_admin(self) -> dict[str, Any] | None:
if not self.config.auth_enabled or self.store.count_users() > 0:
return None
password = self.config.bootstrap_admin_password or secrets.token_urlsafe(18)
validate_password(password)
user = self.store.create_user(
username=self.config.bootstrap_admin_username,
password_hash=self.hash_password(password),
role="admin",
must_change_password=True,
)
if not self.config.bootstrap_admin_password:
_write_bootstrap_file(self.config.auth_bootstrap_path, user["username"], password)
return user
def hash_password(self, password: str) -> str:
validate_password(password)
if self._hasher is None:
raise RuntimeError("authentication is disabled")
return str(self._hasher.hash(password))
def verify_password(self, password_hash: str, password: str) -> bool:
if self._hasher is None:
return False
try:
return bool(self._hasher.verify(password_hash, password))
except Exception:
return False
def login(self, username: str, password: str) -> tuple[dict[str, Any], str]:
user = self.store.get_user_by_username(username.strip())
if not user or user["disabled"] or not self.verify_password(str(user["password_hash"]), password):
raise AuthenticationError("用户名或密码错误")
token = secrets.token_urlsafe(48)
self.store.create_session(
token_hash=token_digest(token),
user_id=user["id"],
expires_at=time.time() + self.config.session_ttl_seconds,
)
self.store.update_user(user["id"], last_login=time.time())
return self.store.get_user(user["id"]), token
def authenticate(self, token: str | None) -> dict[str, Any] | None:
if not self.config.auth_enabled:
return {
"id": "local-development",
"username": "local",
"role": "admin",
"disabled": False,
"must_change_password": False,
}
if not token:
return None
return self.store.get_session_user(token_digest(token))
def logout(self, token: str | None) -> None:
if token:
self.store.delete_session(token_digest(token))
def create_user(self, *, username: str, password: str, role: str) -> dict[str, Any]:
validate_username(username)
validate_role(role)
return self.store.create_user(
username=username.strip(),
password_hash=self.hash_password(password),
role=role,
must_change_password=True,
)
def change_password(self, user_id: str, *, current_password: str | None, new_password: str, admin_reset: bool = False) -> dict[str, Any]:
user_with_hash = self.store.get_user_by_username(self.store.get_user(user_id)["username"])
if not user_with_hash:
raise KeyError(user_id)
if not admin_reset and not self.verify_password(str(user_with_hash["password_hash"]), current_password or ""):
raise AuthenticationError("当前密码不正确")
password_hash = self.hash_password(new_password)
self.store.delete_user_sessions(user_id)
return self.store.update_user(user_id, password_hash=password_hash, must_change_password=admin_reset)
def token_digest(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()
def validate_username(username: str) -> None:
if not _USERNAME_RE.fullmatch(username.strip()):
raise ValueError("用户名需为 3-64 位字母、数字、点、下划线或连字符")
def validate_role(role: str) -> None:
if role not in VALID_ROLES:
raise ValueError("role must be admin, engineer, or viewer")
def validate_password(password: str) -> None:
if len(password) < 12:
raise ValueError("密码至少需要 12 个字符")
classes = [any(c.islower() for c in password), any(c.isupper() for c in password), any(c.isdigit() for c in password), any(not c.isalnum() for c in password)]
if sum(classes) < 3:
raise ValueError("密码需包含大小写字母、数字或符号中的至少三类")
def _password_hasher():
try:
from argon2 import PasswordHasher
from argon2.low_level import Type
except ImportError as exc:
raise RuntimeError("auth_enabled requires argon2-cffi") from exc
return PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16, type=Type.ID)
def _write_bootstrap_file(path: Path, username: str, password: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f"username={username}\npassword={password}\nforce_password_change=true\n", encoding="utf-8")
path.chmod(0o600)

View File

@ -33,6 +33,24 @@ class PlatformConfig:
cuda_home: Path = Path("/usr/local/cuda-12.1")
runtime_env_script: Path = Path("/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh")
allowed_path_roots: tuple[Path, ...] = ()
worker_mode: str = "embedded"
worker_id: str = "local-worker"
production_mode: bool = False
auth_enabled: bool = False
secure_cookies: bool = False
session_ttl_seconds: int = 43200
bootstrap_admin_username: str = "admin"
bootstrap_admin_password: str = ""
bos_endpoint: str = "https://bj.bcebos.com"
bos_bucket: str = ""
bos_prefix: str = "real2sim-poc"
bos_access_key_id: str = ""
bos_secret_access_key: str = ""
discoverse_commit: str = "d67f47c084aba0e0cf422a8725235f8b9238655a"
act_commit: str = "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"
anysplat_commit: str = "5f5e208a7dd57d52e43ea0d553a95eab526e8775"
anysplat_model_revision: str = "d2e8c343672646041ad4ea518184968f94362f01"
vggt_model_revision: str = "860abec7937da0a4c03c41d3c269c366e82abdf9"
@property
def db_path(self) -> Path:
@ -62,6 +80,14 @@ class PlatformConfig:
def manifests_dir(self) -> Path:
return self.workspace_dir / "manifests"
@property
def upload_parts_dir(self) -> Path:
return self.workspace_dir / "upload_parts"
@property
def auth_bootstrap_path(self) -> Path:
return self.workspace_dir / "bootstrap-admin.txt"
@property
def effective_allowed_path_roots(self) -> tuple[Path, ...]:
if self.allowed_path_roots:
@ -80,9 +106,7 @@ class PlatformConfig:
def load_config(path: Path | None = None) -> PlatformConfig:
config_path = path or Path(os.environ.get("REAL2SIM_PLATFORM_CONFIG", ""))
if not config_path or not config_path.is_file():
return PlatformConfig()
values = _parse_simple_toml(config_path.read_text(encoding="utf-8"))
values = _parse_simple_toml(config_path.read_text(encoding="utf-8")) if config_path and config_path.is_file() else {}
app_root = Path(str(values.get("app_root", APP_ROOT)))
workspace_dir = Path(str(values.get("workspace_dir", WORKSPACE_ROOT)))
return PlatformConfig(
@ -106,11 +130,29 @@ def load_config(path: Path | None = None) -> PlatformConfig:
cuda_home=Path(str(values.get("cuda_home", "/usr/local/cuda-12.1"))),
runtime_env_script=Path(str(values.get("runtime_env_script", "/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh"))),
allowed_path_roots=_parse_path_tuple(str(values.get("allowed_path_roots", ""))),
worker_mode=str(values.get("worker_mode", os.environ.get("REAL2SIM_WORKER_MODE", "embedded"))),
worker_id=str(values.get("worker_id", os.environ.get("REAL2SIM_WORKER_ID", "local-worker"))),
production_mode=_as_bool(values.get("production_mode", os.environ.get("REAL2SIM_PRODUCTION_MODE", "0"))),
auth_enabled=_as_bool(values.get("auth_enabled", os.environ.get("REAL2SIM_AUTH_ENABLED", "0"))),
secure_cookies=_as_bool(values.get("secure_cookies", os.environ.get("REAL2SIM_SECURE_COOKIES", "0"))),
session_ttl_seconds=int(values.get("session_ttl_seconds", os.environ.get("REAL2SIM_SESSION_TTL_SECONDS", "43200"))),
bootstrap_admin_username=str(values.get("bootstrap_admin_username", os.environ.get("REAL2SIM_BOOTSTRAP_ADMIN_USERNAME", "admin"))),
bootstrap_admin_password=str(values.get("bootstrap_admin_password", os.environ.get("REAL2SIM_BOOTSTRAP_ADMIN_PASSWORD", ""))),
bos_endpoint=str(values.get("bos_endpoint", os.environ.get("BOS_ENDPOINT", "https://bj.bcebos.com"))),
bos_bucket=str(values.get("bos_bucket", os.environ.get("BOS_BUCKET", ""))),
bos_prefix=str(values.get("bos_prefix", os.environ.get("REAL2SIM_BOS_PREFIX", "real2sim-poc"))),
bos_access_key_id=str(values.get("bos_access_key_id", os.environ.get("BOS_ACCESS_KEY_ID", ""))),
bos_secret_access_key=str(values.get("bos_secret_access_key", os.environ.get("BOS_SECRET_ACCESS_KEY", ""))),
discoverse_commit=str(values.get("discoverse_commit", os.environ.get("REAL2SIM_DISCOVERSE_COMMIT", "d67f47c084aba0e0cf422a8725235f8b9238655a"))),
act_commit=str(values.get("act_commit", os.environ.get("REAL2SIM_ACT_COMMIT", "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"))),
anysplat_commit=str(values.get("anysplat_commit", os.environ.get("REAL2SIM_ANYSPLAT_COMMIT", "5f5e208a7dd57d52e43ea0d553a95eab526e8775"))),
anysplat_model_revision=str(values.get("anysplat_model_revision", os.environ.get("REAL2SIM_ANYSPLAT_MODEL_REVISION", "d2e8c343672646041ad4ea518184968f94362f01"))),
vggt_model_revision=str(values.get("vggt_model_revision", os.environ.get("REAL2SIM_VGGT_MODEL_REVISION", "860abec7937da0a4c03c41d3c269c366e82abdf9"))),
)
def ensure_workspace(config: PlatformConfig) -> None:
for path in [config.workspace_dir, config.uploads_dir, config.datasets_dir, config.scenes_dir, config.runs_dir, config.models_dir, config.manifests_dir]:
for path in [config.workspace_dir, config.uploads_dir, config.upload_parts_dir, config.datasets_dir, config.scenes_dir, config.runs_dir, config.models_dir, config.manifests_dir]:
path.mkdir(parents=True, exist_ok=True)
@ -138,6 +180,21 @@ def write_default_config(path: Path, config: PlatformConfig | None = None) -> No
f'cuda_home = "{cfg.cuda_home.as_posix()}"',
f'runtime_env_script = "{cfg.runtime_env_script.as_posix()}"',
f'allowed_path_roots = "{_format_path_tuple(cfg.effective_allowed_path_roots)}"',
f'worker_mode = "{cfg.worker_mode}"',
f'worker_id = "{cfg.worker_id}"',
f"production_mode = {str(cfg.production_mode).lower()}",
f"auth_enabled = {str(cfg.auth_enabled).lower()}",
f"secure_cookies = {str(cfg.secure_cookies).lower()}",
f"session_ttl_seconds = {cfg.session_ttl_seconds}",
f'bootstrap_admin_username = "{cfg.bootstrap_admin_username}"',
f'bos_endpoint = "{cfg.bos_endpoint}"',
f'bos_bucket = "{cfg.bos_bucket}"',
f'bos_prefix = "{cfg.bos_prefix}"',
f'discoverse_commit = "{cfg.discoverse_commit}"',
f'act_commit = "{cfg.act_commit}"',
f'anysplat_commit = "{cfg.anysplat_commit}"',
f'anysplat_model_revision = "{cfg.anysplat_model_revision}"',
f'vggt_model_revision = "{cfg.vggt_model_revision}"',
"",
]
path.write_text("\n".join(lines), encoding="utf-8")
@ -169,6 +226,9 @@ def _parse_simple_toml(text: str) -> dict[str, Any]:
if value.startswith('"') and value.endswith('"'):
values[key] = value[1:-1]
else:
if value.lower() in {"true", "false"}:
values[key] = value.lower() == "true"
continue
try:
values[key] = int(value)
except ValueError:
@ -184,3 +244,9 @@ def _parse_path_tuple(value: str) -> tuple[Path, ...]:
def _format_path_tuple(paths: tuple[Path, ...]) -> str:
return ":".join(path.as_posix() for path in paths)
def _as_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
return str(value).strip().lower() in {"1", "true", "yes", "on"}

View File

@ -0,0 +1,379 @@
from __future__ import annotations
import argparse
import json
import math
import subprocess
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import numpy as np
@dataclass
class EpisodeResult:
success: bool
lift_height: float
place_error: float
bilateral_contact_steps: int
observations: list[np.ndarray]
actions: list[np.ndarray]
contacts: list[np.ndarray]
images: list[np.ndarray]
trajectory: list[dict[str, Any]]
initial_cube: list[float]
target: list[float]
class RM65PickPlaceRuntime:
def __init__(self, scene_path: Path, *, width: int, height: int, seed: int) -> None:
import mujoco
self.mujoco = mujoco
self.model = mujoco.MjModel.from_xml_path(str(scene_path))
self.data = mujoco.MjData(self.model)
self.renderer = mujoco.Renderer(self.model, height=height, width=width)
self.rng = np.random.default_rng(seed)
self.site_id = self._id(mujoco.mjtObj.mjOBJ_SITE, "gripper")
self.cube_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "pick_cube_free")
self.cube_qadr = int(self.model.jnt_qposadr[self.cube_joint])
self.cube_geom = self._id(mujoco.mjtObj.mjOBJ_GEOM, "pick_cube_geom")
self.finger_geoms = {
self._id(mujoco.mjtObj.mjOBJ_GEOM, "left_finger_pad_z_axis"),
self._id(mujoco.mjtObj.mjOBJ_GEOM, "right_finger_pad_z_axis"),
}
self.target_body = self._id(mujoco.mjtObj.mjOBJ_BODY, "place_target")
self.gripper_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint")
self.gripper_qadr = int(self.model.jnt_qposadr[self.gripper_joint])
self.joint_ids = [self._id(mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}") for index in range(1, 7)]
self.qpos_adrs = [int(self.model.jnt_qposadr[joint]) for joint in self.joint_ids]
self.dof_adrs = [int(self.model.jnt_dofadr[joint]) for joint in self.joint_ids]
self.ctrl_ranges = self.model.actuator_ctrlrange.copy()
def _id(self, kind: Any, name: str) -> int:
value = self.mujoco.mj_name2id(self.model, kind, name)
if value < 0:
raise ValueError(f"required RM65 scene object is missing: {name}")
return int(value)
def close(self) -> None:
self.renderer.close()
def run_episode(self, *, max_steps: int) -> EpisodeResult:
mujoco = self.mujoco
mujoco.mj_resetDataKeyframe(self.model, self.data, 0)
cube = np.array([self.rng.uniform(0.645, 0.655), self.rng.uniform(-0.010, 0.010), 0.33], dtype=float)
target = np.array([self.rng.uniform(0.42, 0.51), self.rng.uniform(0.13, 0.22), 0.31], dtype=float)
self.data.qpos[self.cube_qadr : self.cube_qadr + 3] = cube
self.data.qpos[self.cube_qadr + 3 : self.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
self.data.qvel[self.model.jnt_dofadr[self.cube_joint] : self.model.jnt_dofadr[self.cube_joint] + 6] = 0
self.model.body_pos[self.target_body] = np.array([target[0], target[1], 0.335])
mujoco.mj_forward(self.model, self.data)
for _ in range(120):
mujoco.mj_step(self.model, self.data)
settled_z = float(self.data.qpos[self.cube_qadr + 2])
# The named site is about 3 cm above the cube center at bilateral pad contact.
grasp_site = np.array([cube[0] - 0.014, cube[1] - 0.020, settled_z + 0.020])
# Contact-calibrated negative-wrist branch for the packaged RM65 gripper.
# This seed is a robot calibration pose, not an object-pose override.
seed_q = np.array([-1.2468, -0.1011, 1.8666, -1.9230, -1.5982, 1.5636], dtype=float)
q_backup = self.data.qpos.copy()
self.data.qpos[self.qpos_adrs] = seed_q
mujoco.mj_forward(self.model, self.data)
grasp_orientation = self.data.site_xmat[self.site_id].reshape(3, 3).copy()
self.data.qpos[:] = q_backup
mujoco.mj_forward(self.model, self.data)
phases = [
("approach", grasp_site + np.array([0.0, 0.0, 0.20]), 0.0, 24),
("descend", grasp_site, 0.0, 24),
("grasp", grasp_site, -0.28, 30),
("lift", grasp_site + np.array([0.0, 0.0, 0.20]), -0.28, 60),
("transfer", target + np.array([-0.040, 0.0, 0.21]), -0.28, 34),
("place", target + np.array([-0.040, 0.0, 0.02]), -0.28, 26),
("release", target + np.array([-0.040, 0.0, 0.02]), 0.0, 18),
("retreat", target + np.array([-0.040, 0.0, 0.20]), 0.0, 20),
]
scale = max(0.25, max_steps / sum(phase[3] for phase in phases))
observations: list[np.ndarray] = []
actions: list[np.ndarray] = []
contacts: list[np.ndarray] = []
images: list[np.ndarray] = []
trajectory: list[dict[str, Any]] = []
bilateral_contact_steps = 0
max_cube_z = settled_z
step_index = 0
for phase, position, gripper, nominal_steps in phases:
target_q = self.solve_ik_6d(position, grasp_orientation, seed_q=seed_q)
phase_steps = max(2, int(round(nominal_steps * scale)))
start_ctrl = self.data.ctrl[:6].copy()
for local_step in range(phase_steps):
alpha = _smoothstep((local_step + 1) / phase_steps)
action = np.empty(7, dtype=np.float32)
action[:6] = start_ctrl * (1 - alpha) + target_q * alpha
action[6] = gripper
self.data.ctrl[:7] = action
# The RM65 model has conservative actuator force limits; each policy
# command is held long enough for the physical joints to settle.
for _ in range(35):
mujoco.mj_step(self.model, self.data)
touching = self.finger_contacts()
bilateral_contact_steps += int(len(touching) == 2)
max_cube_z = max(max_cube_z, float(self.data.qpos[self.cube_qadr + 2]))
self.renderer.update_scene(self.data, camera="overview")
frame = self.renderer.render().copy()
observation = np.concatenate([self.data.qpos[self.qpos_adrs], [self.data.qpos[self.gripper_qadr]]]).astype(np.float32)
observations.append(observation)
actions.append(action.copy())
contacts.append(np.array([int(geom in touching) for geom in sorted(self.finger_geoms)], dtype=np.uint8))
images.append(frame)
trajectory.append(
{
"step": step_index,
"time": float(self.data.time),
"phase": phase,
"qpos": observation.tolist(),
"action": action.tolist(),
"cube_pos": self.data.qpos[self.cube_qadr : self.cube_qadr + 3].astype(float).tolist(),
"gripper_pos": self.data.site_xpos[self.site_id].astype(float).tolist(),
"bilateral_contact": len(touching) == 2,
"touching_fingers": sorted(int(item) for item in touching),
"finger_positions": [self.data.geom_xpos[item].astype(float).tolist() for item in sorted(self.finger_geoms)],
}
)
step_index += 1
final_cube = self.data.qpos[self.cube_qadr : self.cube_qadr + 3].copy()
lift_height = float(max_cube_z - settled_z)
place_error = float(np.linalg.norm(final_cube - target))
success = bilateral_contact_steps > 0 and lift_height >= 0.10 and place_error <= 0.05
return EpisodeResult(
success=success,
lift_height=lift_height,
place_error=place_error,
bilateral_contact_steps=bilateral_contact_steps,
observations=observations,
actions=actions,
contacts=contacts,
images=images,
trajectory=trajectory,
initial_cube=cube.tolist(),
target=target.tolist(),
)
def finger_contacts(self) -> set[int]:
touching: set[int] = set()
for index in range(self.data.ncon):
contact = self.data.contact[index]
pair = {int(contact.geom1), int(contact.geom2)}
if self.cube_geom in pair:
touching.update(pair & self.finger_geoms)
return touching
def solve_position_ik(self, target_position: np.ndarray) -> np.ndarray:
q_backup = self.data.qpos.copy()
q = self.data.qpos[self.qpos_adrs].copy()
for _ in range(120):
self.data.qpos[self.qpos_adrs] = q
self.mujoco.mj_forward(self.model, self.data)
error = target_position - self.data.site_xpos[self.site_id]
if np.linalg.norm(error) < 0.004:
break
jacp = np.zeros((3, self.model.nv))
jacr = np.zeros((3, self.model.nv))
self.mujoco.mj_jacSite(self.model, self.data, jacp, jacr, self.site_id)
jacobian = jacp[:, self.dof_adrs]
q += np.clip(_damped_least_squares(jacobian, error, 2e-3), -0.06, 0.06)
q = self._clip_arm(q)
self.data.qpos[:] = q_backup
self.mujoco.mj_forward(self.model, self.data)
return q
def solve_ik_6d(self, target_position: np.ndarray, target_rotation: np.ndarray, *, seed_q: np.ndarray | None = None) -> np.ndarray:
q_backup = self.data.qpos.copy()
q = seed_q.copy() if seed_q is not None else self.data.qpos[self.qpos_adrs].copy()
for _ in range(160):
self.data.qpos[self.qpos_adrs] = q
self.mujoco.mj_forward(self.model, self.data)
position_error = target_position - self.data.site_xpos[self.site_id]
current_rotation = self.data.site_xmat[self.site_id].reshape(3, 3)
rotation_error = _rotation_vector(target_rotation @ current_rotation.T)
error = np.concatenate([position_error, 0.45 * rotation_error])
if np.linalg.norm(position_error) < 0.005 and np.linalg.norm(rotation_error) < 0.04:
break
jacp = np.zeros((3, self.model.nv))
jacr = np.zeros((3, self.model.nv))
self.mujoco.mj_jacSite(self.model, self.data, jacp, jacr, self.site_id)
jacobian = np.vstack([jacp[:, self.dof_adrs], 0.45 * jacr[:, self.dof_adrs]])
q += np.clip(_damped_least_squares(jacobian, error, 4e-3), -0.05, 0.05)
q = self._clip_arm(q)
self.data.qpos[:] = q_backup
self.mujoco.mj_forward(self.model, self.data)
return q
def _clip_arm(self, q: np.ndarray) -> np.ndarray:
return np.array([np.clip(q[index], self.ctrl_ranges[index, 0], self.ctrl_ranges[index, 1]) for index in range(6)])
def collect(scene_path: Path, out_dir: Path, *, required_successes: int, max_attempts: int, max_steps: int, seed: int, width: int, height: int) -> dict[str, Any]:
out_dir.mkdir(parents=True, exist_ok=True)
runtime = RM65PickPlaceRuntime(scene_path, width=width, height=height, seed=seed)
reports: list[dict[str, Any]] = []
success_count = 0
try:
for attempt in range(max_attempts):
result = runtime.run_episode(max_steps=max_steps)
episode_report = {
"attempt": attempt,
"success": result.success,
"lift_height": result.lift_height,
"place_error": result.place_error,
"bilateral_contact_steps": result.bilateral_contact_steps,
"initial_cube": result.initial_cube,
"target": result.target,
}
reports.append(episode_report)
trajectory_path = out_dir / f"attempt_{attempt:04d}_trajectory.json"
trajectory_path.write_text(json.dumps({"schema": "real2sim.rm65.trajectory.v2", "report": episode_report, "samples": result.trajectory}, ensure_ascii=False) + "\n", encoding="utf-8")
if result.success:
_write_hdf5_episode(out_dir / f"episode_{success_count}.hdf5", result)
if success_count < 3:
_write_preview_video(out_dir / f"episode_{success_count}.mp4", result.images)
success_count += 1
if success_count >= required_successes:
break
finally:
runtime.close()
manifest = {
"schema": "real2sim.discoverse_act_dataset.v1",
"controller": "rm65_pick_place_6d_ik",
"runtime": "mujoco",
"scene": str(scene_path),
"camera_names": ["overview"],
"state_dim": 7,
"action_dim": 7,
"required_successes": required_successes,
"successful_episodes": success_count,
"attempts": len(reports),
"success_rate": success_count / max(1, len(reports)),
"success_definition": {"minimum_lift_m": 0.10, "maximum_place_error_m": 0.05, "bilateral_contact_required": True},
"episodes": reports,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
(out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return manifest
def _write_hdf5_episode(path: Path, result: EpisodeResult) -> None:
import cv2
import h5py
observations = np.asarray(result.observations, dtype=np.float32)
actions = np.asarray(result.actions, dtype=np.float32)
contacts = np.asarray(result.contacts, dtype=np.uint8)
encoded = []
for image in result.images:
ok, buffer = cv2.imencode(".jpg", image[:, :, ::-1], [int(cv2.IMWRITE_JPEG_QUALITY), 88])
if not ok:
raise RuntimeError("failed to encode collection frame")
encoded.append(buffer.astype(np.uint8))
with h5py.File(path, "w") as root:
root.attrs["sim"] = True
root.attrs["compress"] = True
root.attrs["success"] = bool(result.success)
root.create_dataset("/observations/qpos", data=observations, dtype="f4")
root.create_dataset("/action", data=actions, dtype="f4")
root.create_dataset("/observations/contact", data=contacts, dtype="u1")
root.create_dataset("/success", data=np.full(len(actions), result.success, dtype=np.uint8))
variable = h5py.vlen_dtype(np.dtype("uint8"))
dataset = root.create_dataset("/observations/images/overview", (len(encoded),), dtype=variable)
for index, value in enumerate(encoded):
dataset[index] = value
def _write_preview_video(path: Path, images: list[np.ndarray]) -> None:
if not images:
raise ValueError("cannot encode an empty rollout")
height, width = images[0].shape[:2]
command = [
"ffmpeg",
"-y",
"-loglevel",
"error",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
"-s",
f"{width}x{height}",
"-r",
"20",
"-i",
"-",
"-an",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
str(path),
]
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
assert process.stdin is not None
for image in images:
process.stdin.write(np.ascontiguousarray(image, dtype=np.uint8).tobytes())
process.stdin.close()
stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else ""
if process.wait() != 0 or not path.is_file() or path.stat().st_size == 0:
raise RuntimeError(f"ffmpeg rollout encoding failed: {stderr[-2000:]}")
def _damped_least_squares(jacobian: np.ndarray, error: np.ndarray, damping: float) -> np.ndarray:
identity = np.eye(jacobian.shape[0])
return jacobian.T @ np.linalg.solve(jacobian @ jacobian.T + damping * identity, error)
def _rotation_vector(rotation: np.ndarray) -> np.ndarray:
cosine = float(np.clip((np.trace(rotation) - 1.0) * 0.5, -1.0, 1.0))
angle = math.acos(cosine)
if angle < 1e-7:
return np.zeros(3)
axis = np.array([rotation[2, 1] - rotation[1, 2], rotation[0, 2] - rotation[2, 0], rotation[1, 0] - rotation[0, 1]])
denominator = 2.0 * math.sin(angle)
return axis * (angle / denominator)
def _smoothstep(value: float) -> float:
value = float(np.clip(value, 0.0, 1.0))
return value * value * (3.0 - 2.0 * value)
def main() -> int:
parser = argparse.ArgumentParser(description="Collect physical RM65 pick-place episodes for DISCOVERSE ACT")
parser.add_argument("--scene", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--required-successes", type=int, default=50)
parser.add_argument("--max-attempts", type=int, default=150)
parser.add_argument("--steps", type=int, default=200)
parser.add_argument("--seed", type=int, default=7)
parser.add_argument("--width", type=int, default=320)
parser.add_argument("--height", type=int, default=240)
args = parser.parse_args()
manifest = collect(
args.scene,
args.out,
required_successes=args.required_successes,
max_attempts=args.max_attempts,
max_steps=args.steps,
seed=args.seed,
width=args.width,
height=args.height,
)
print(json.dumps(manifest, ensure_ascii=False, indent=2))
return 0 if manifest["successful_episodes"] >= manifest["required_successes"] else 2
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,196 @@
from __future__ import annotations
import argparse
import gc
import hashlib
import json
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from typing import Any
from huggingface_hub import snapshot_download
from .adapters import run_doctor
from .config import PlatformConfig, ensure_workspace, load_config
def main() -> int:
parser = argparse.ArgumentParser(description="Validate the production Real2Sim GPU runtime")
parser.add_argument("--mode", choices=["bootstrap", "readiness"], default="readiness")
parser.add_argument("--force", action="store_true")
args = parser.parse_args()
config = load_config()
ensure_workspace(config)
marker = marker_path(config)
if args.mode == "readiness":
return readiness(config, marker)
if marker.is_file() and not args.force:
try:
existing = json.loads(marker.read_text(encoding="utf-8"))
if existing.get("runtime_key") == runtime_key(config) and existing.get("ok") is True:
print(json.dumps(existing, ensure_ascii=False))
return 0
except (OSError, json.JSONDecodeError):
pass
report = bootstrap(config)
marker.parent.mkdir(parents=True, exist_ok=True)
temporary = marker.with_suffix(".tmp")
temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(marker)
print(json.dumps(report, ensure_ascii=False))
return 0 if report["ok"] else 1
def marker_path(config: PlatformConfig) -> Path:
root = Path(os.environ.get("REAL2SIM_RUNTIME_GATE_DIR", config.workspace_dir / "runtime_gate"))
return root / f"{runtime_key(config)}.json"
def runtime_key(config: PlatformConfig) -> str:
raw = ":".join(
[
config.anysplat_commit,
config.anysplat_model_revision,
config.vggt_model_revision,
config.discoverse_commit,
config.act_commit,
]
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:20]
def readiness(config: PlatformConfig, marker: Path) -> int:
doctor = run_doctor(config)
if not marker.is_file():
print("runtime bootstrap report is missing", file=sys.stderr)
return 1
try:
report = json.loads(marker.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
print(f"runtime bootstrap report is unreadable: {exc}", file=sys.stderr)
return 1
ready = bool(report.get("ok") and report.get("runtime_key") == runtime_key(config) and doctor.get("ok"))
if not ready:
print(json.dumps({"bootstrap": report, "doctor": doctor}, ensure_ascii=False), file=sys.stderr)
return 0 if ready else 1
def bootstrap(config: PlatformConfig) -> dict[str, Any]:
started = time.time()
checks: dict[str, Any] = {}
doctor = run_doctor(config)
checks["doctor"] = {"ok": doctor.get("ok"), "core_checks": doctor.get("core_checks"), "checks": doctor.get("checks")}
try:
checks["nvidia"] = _nvidia_probe()
checks["simulators"] = _simulator_probe()
checks["act"] = _act_probe(config)
checks["anysplat"] = _anysplat_probe(config)
except Exception as exc:
checks["failure"] = {"type": type(exc).__name__, "message": str(exc)}
ok = bool(doctor.get("ok")) and all(bool(value.get("ok")) for key, value in checks.items() if key != "failure") and "failure" not in checks
return {
"schema": "real2sim.runtime_gate.v1",
"runtime_key": runtime_key(config),
"ok": ok,
"started_at": started,
"finished_at": time.time(),
"checks": checks,
"revisions": {
"anysplat_code": config.anysplat_commit,
"anysplat_model": config.anysplat_model_revision,
"vggt_model": config.vggt_model_revision,
"discoverse": config.discoverse_commit,
"act": config.act_commit,
},
}
def _nvidia_probe() -> dict[str, Any]:
completed = subprocess.run(
["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"],
text=True,
capture_output=True,
timeout=20,
check=True,
)
import torch
if not torch.cuda.is_available():
raise RuntimeError("PyTorch CUDA is unavailable")
value = torch.ones((1024, 1024), device="cuda")
checksum = float((value @ value).mean().item())
del value
torch.cuda.empty_cache()
return {"ok": checksum == 1024.0, "device": torch.cuda.get_device_name(0), "nvidia_smi": completed.stdout.strip().splitlines()[0]}
def _simulator_probe() -> dict[str, Any]:
xml = """<mujoco model="runtime-gate"><option timestep="0.002"/><worldbody><body pos="0 0 0.2"><freejoint/><geom type="sphere" size="0.03" mass="0.1"/></body><geom type="plane" size="1 1 .1"/></worldbody></mujoco>"""
import mujoco
model = mujoco.MjModel.from_xml_string(xml)
data = mujoco.MjData(model)
for _ in range(8):
mujoco.mj_step(model, data)
with tempfile.TemporaryDirectory(prefix="real2sim-runtime-gate-") as temporary:
scene_path = Path(temporary) / "scene.xml"
scene_path.write_text(xml, encoding="utf-8")
from motrixsim import SceneData, load_model, step
motrix_model = load_model(str(scene_path))
motrix_data = SceneData(motrix_model)
for _ in range(8):
step(motrix_model, motrix_data)
return {"ok": True, "mujoco_time": float(data.time), "motrix_timestep": float(motrix_model.options.timestep)}
def _act_probe(config: PlatformConfig) -> dict[str, Any]:
act_root = config.discoverse_root / "policies" / "act"
env = os.environ.copy()
env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(config.discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep)
completed = subprocess.run(
[sys.executable, "-c", "import torch; from policy import ACTPolicy; print(torch.__version__); assert ACTPolicy is not None and torch.cuda.is_available()"],
cwd=act_root,
env=env,
text=True,
capture_output=True,
timeout=60,
check=True,
)
return {"ok": True, "torch": completed.stdout.strip().splitlines()[-1]}
def _anysplat_probe(config: PlatformConfig) -> dict[str, Any]:
cache_dir = Path(os.environ.get("HF_HOME", config.workspace_dir / "cache" / "huggingface"))
cache_dir.mkdir(parents=True, exist_ok=True)
anysplat_snapshot = Path(
snapshot_download(
repo_id="lhjiang/anysplat",
revision=config.anysplat_model_revision,
cache_dir=cache_dir,
)
)
snapshot_download(repo_id="facebook/VGGT-1B", revision=config.vggt_model_revision, cache_dir=cache_dir)
if str(config.anysplat_root) not in sys.path:
sys.path.insert(0, str(config.anysplat_root))
from src.model.model.anysplat import AnySplat
model = AnySplat.from_pretrained(str(anysplat_snapshot))
model = model.to("cuda").eval()
parameters = sum(parameter.numel() for parameter in model.parameters())
if parameters <= 0:
raise RuntimeError("AnySplat model loaded with zero parameters")
del model
gc.collect()
import torch
torch.cuda.empty_cache()
return {"ok": True, "parameter_count": parameters, "snapshot": anysplat_snapshot.name}
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -1,24 +1,33 @@
from __future__ import annotations
import argparse
import asyncio
from collections import defaultdict, deque
import hashlib
import json
import os
import math
import secrets
import shutil
import subprocess
import threading
import time
import zipfile
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
from fastapi import FastAPI, File, HTTPException, UploadFile
from fastapi.responses import FileResponse, PlainTextResponse
from fastapi import FastAPI, File, HTTPException, Request, Response, UploadFile
from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from . import __version__
from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset
from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username
from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config
from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file
from .store import PlatformStore
from .worker import TaskWorker
@ -63,24 +72,95 @@ class BulkActionRequest(BaseModel):
params: dict[str, Any] = {}
class LoginRequest(BaseModel):
username: str
password: str
class ChangePasswordRequest(BaseModel):
current_password: str = ""
new_password: str
class CreateUserRequest(BaseModel):
username: str
password: str
role: str = "engineer"
class UpdateUserRequest(BaseModel):
role: str | None = None
disabled: bool | None = None
reset_password: str | None = None
class CreateUploadRequest(BaseModel):
filename: str
content_type: str = "application/octet-stream"
total_size: int
chunk_size: int = 8 * 1024 * 1024
checksum: str = ""
class ImportBosRequest(BaseModel):
prefix: str
name: str | None = None
max_objects: int = 1000
def create_app(config: PlatformConfig | None = None) -> FastAPI:
cfg = config or load_config()
ensure_workspace(cfg)
store = PlatformStore(cfg.db_path)
worker = TaskWorker(store, cfg)
app = FastAPI(title="Real2Sim Platform", version="0.1.0")
auth = AuthManager(store, cfg)
storage = BosStorage(cfg)
auth.bootstrap_admin()
login_failures: dict[str, deque[float]] = defaultdict(deque)
login_failure_lock = threading.Lock()
@asynccontextmanager
async def lifespan(_app: FastAPI):
if cfg.worker_mode == "embedded":
worker.start()
try:
yield
finally:
if cfg.worker_mode == "embedded":
worker.stop()
app = FastAPI(title="Real2Sim Platform", version="0.1.0", lifespan=lifespan)
static_dir = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=static_dir), name="static")
app.mount("/datasets", StaticFiles(directory=cfg.datasets_dir), name="datasets")
@app.on_event("startup")
def _startup() -> None:
store.recover_interrupted_tasks()
worker.start()
public_paths = {"/", "/api/health", "/api/version", "/api/capabilities", "/api/auth/login"}
@app.on_event("shutdown")
def _shutdown() -> None:
worker.stop()
@app.middleware("http")
async def authenticate_request(request: Request, call_next):
path = request.url.path
if not cfg.auth_enabled or path in public_paths or path.startswith("/static/"):
request.state.user = auth.authenticate(None)
return await call_next(request)
token = request.cookies.get(SESSION_COOKIE)
user = auth.authenticate(token)
if user is None:
return JSONResponse(status_code=401, content={"detail": "请先登录"})
request.state.user = user
if user.get("must_change_password") and path not in {"/api/auth/me", "/api/auth/change-password", "/api/auth/logout"}:
return JSONResponse(status_code=428, content={"detail": "首次登录必须先修改密码", "code": "password_change_required"})
if request.method not in {"GET", "HEAD", "OPTIONS"}:
self_service_paths = {"/api/auth/change-password", "/api/auth/logout"}
if user.get("role") == "viewer" and path not in self_service_paths:
return JSONResponse(status_code=403, content={"detail": "只读用户不能执行修改操作"})
if path.startswith("/api/users") and user.get("role") != "admin":
return JSONResponse(status_code=403, content={"detail": "仅管理员可以管理用户"})
origin = request.headers.get("origin")
origin_host = urlparse(origin).hostname if origin else None
if origin_host and origin_host != request.url.hostname:
return JSONResponse(status_code=403, content={"detail": "请求来源校验失败"})
if path.startswith("/api/users") and user.get("role") != "admin":
return JSONResponse(status_code=403, content={"detail": "仅管理员可以管理用户"})
return await call_next(request)
@app.get("/")
def index():
@ -88,7 +168,110 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
@app.get("/api/health")
def health() -> dict[str, Any]:
return {"ok": True, "workspace": str(cfg.workspace_dir)}
return {"ok": True, "worker_mode": cfg.worker_mode, "production_mode": cfg.production_mode}
@app.post("/api/auth/login")
def login(payload: LoginRequest, request: Request, response: Response) -> dict[str, Any]:
if not cfg.auth_enabled:
return {"user": auth.authenticate(None), "auth_enabled": False}
login_key = f"{request.client.host if request.client else 'unknown'}:{payload.username.strip().lower()}"
now = time.time()
with login_failure_lock:
attempts = login_failures[login_key]
while attempts and now - attempts[0] > 900:
attempts.popleft()
if len(attempts) >= 8:
raise HTTPException(status_code=429, detail="登录失败次数过多,请 15 分钟后再试")
try:
user, token = auth.login(payload.username, payload.password)
except AuthenticationError as exc:
with login_failure_lock:
login_failures[login_key].append(time.time())
time.sleep(0.2)
raise HTTPException(status_code=401, detail=str(exc)) from exc
with login_failure_lock:
login_failures.pop(login_key, None)
response.set_cookie(
SESSION_COOKIE,
token,
max_age=cfg.session_ttl_seconds,
httponly=True,
secure=cfg.secure_cookies,
samesite="lax",
path="/",
)
return {"user": user, "auth_enabled": True}
@app.post("/api/auth/logout")
def logout(request: Request, response: Response) -> dict[str, bool]:
auth.logout(request.cookies.get(SESSION_COOKIE))
response.delete_cookie(SESSION_COOKIE, path="/")
return {"ok": True}
@app.get("/api/auth/me")
def auth_me(request: Request) -> dict[str, Any]:
return {"user": request.state.user, "auth_enabled": cfg.auth_enabled}
@app.post("/api/auth/change-password")
def change_password(payload: ChangePasswordRequest, request: Request, response: Response) -> dict[str, Any]:
try:
user = auth.change_password(
request.state.user["id"],
current_password=payload.current_password,
new_password=payload.new_password,
)
except (AuthenticationError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
response.delete_cookie(SESSION_COOKIE, path="/")
return {"ok": True, "user": user, "login_required": True}
@app.get("/api/users")
def list_users() -> list[dict[str, Any]]:
return store.list_users()
@app.post("/api/users")
def create_user(payload: CreateUserRequest) -> dict[str, Any]:
try:
return auth.create_user(username=payload.username, password=payload.password, role=payload.role)
except (ValueError, __import__("sqlite3").IntegrityError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.patch("/api/users/{user_id}")
def update_user(user_id: str, payload: UpdateUserRequest, request: Request) -> dict[str, Any]:
try:
target = store.get_user(user_id)
if payload.role is not None:
validate_role(payload.role)
if target["id"] == request.state.user["id"] and payload.disabled:
raise ValueError("不能禁用当前登录账号")
if target["role"] == "admin" and (payload.disabled or payload.role not in {None, "admin"}) and _active_admin_count(store) <= 1:
raise ValueError("系统至少需要一个可用管理员")
if payload.reset_password:
validate_password(payload.reset_password)
return auth.change_password(user_id, current_password=None, new_password=payload.reset_password, admin_reset=True)
result = store.update_user(user_id, role=payload.role, disabled=payload.disabled)
if payload.disabled:
store.delete_user_sessions(user_id)
return result
except KeyError as exc:
raise HTTPException(status_code=404, detail="用户不存在") from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.delete("/api/users/{user_id}")
def delete_user(user_id: str, request: Request) -> dict[str, bool]:
try:
target = store.get_user(user_id)
if target["id"] == request.state.user["id"]:
raise ValueError("不能删除当前登录账号")
if target["role"] == "admin" and not target["disabled"] and _active_admin_count(store) <= 1:
raise ValueError("系统至少需要一个可用管理员")
store.delete_user(user_id)
return {"ok": True}
except KeyError as exc:
raise HTTPException(status_code=404, detail="用户不存在") from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.get("/api/version")
def version() -> dict[str, Any]:
@ -120,7 +303,227 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
@app.get("/api/doctor")
def doctor() -> dict[str, Any]:
return run_doctor(cfg)
result = run_doctor(cfg)
bos_health = storage.health()
sqlite_wal = _sqlite_journal_mode(store) == "wal"
result["checks"]["bos_storage"] = bos_health
result["checks"]["sqlite_wal"] = sqlite_wal
result["checks"]["worker_mode"] = {"ready": cfg.worker_mode in {"embedded", "api", "worker"}, "value": cfg.worker_mode}
heartbeats = store.list_worker_heartbeats()
latest = heartbeats[0] if heartbeats else None
heartbeat_age = time.time() - float(latest["updated_at"]) if latest else None
worker_ready = bool(latest and heartbeat_age is not None and heartbeat_age < 30 and latest["payload"].get("ready"))
result["worker"] = {"ready": worker_ready, "age_seconds": heartbeat_age, "latest": latest}
if cfg.production_mode and cfg.worker_mode == "api":
result["ok"] = bool(bos_health.get("ready") and sqlite_wal and worker_ready)
result["core_checks"] = ["bos_storage", "sqlite_wal", "worker"]
return result
@app.get("/api/capabilities")
def capabilities() -> dict[str, Any]:
return _capabilities(cfg, storage, store)
@app.get("/api/recipes")
def recipes() -> dict[str, Any]:
capabilities_payload = _capabilities(cfg, storage, store)
act_ready = bool(capabilities_payload["runtimes"]["discoverse_act"]["ready"])
rm65_ready = bool(capabilities_payload["runtimes"]["rm65_controller"]["ready"])
return {
"collection": [
{
"id": "rm65_pick_place_6d_ik",
"name": "RM65 茶几抓取放置",
"enabled": rm65_ready,
"engine": "MuJoCo / MotrixSim",
"requirements": ["rm65", "parallel_gripper", "pick_place_target", "contact_sensors"],
"reason": "" if rm65_ready else "RM65 物理控制器或运行时尚未安装",
}
],
"training": [
{
"id": "discoverse_act",
"name": "DISCOVERSE ACT",
"enabled": act_ready,
"commit": cfg.act_commit,
"input_format": "discoverse_act_hdf5",
"reason": "" if act_ready else "DISCOVERSE ACT 固定版本运行时尚未就绪",
},
{"id": "bc_v2", "name": "历史岭回归模型", "enabled": False, "read_only": True, "reason": "仅用于查看历史模型,禁止新建"},
],
"inference": [
{"id": "discoverse_act", "name": "DISCOVERSE ACT 在线推理", "enabled": act_ready, "requires_real_simulator": True}
],
}
@app.post("/api/uploads")
def create_upload(payload: CreateUploadRequest, request: Request) -> dict[str, Any]:
if not storage.configured:
raise HTTPException(status_code=503, detail="BOS 存储尚未配置")
if payload.total_size <= 0:
raise HTTPException(status_code=400, detail="total_size must be > 0")
if not 1024 * 1024 <= payload.chunk_size <= 64 * 1024 * 1024:
raise HTTPException(status_code=400, detail="chunk_size must be between 1 MiB and 64 MiB")
if payload.checksum and (len(payload.checksum) != 64 or any(c not in "0123456789abcdefABCDEF" for c in payload.checksum)):
raise HTTPException(status_code=400, detail="checksum must be a SHA-256 hex digest")
filename = safe_object_name(payload.filename)
upload_id = secrets.token_hex(16)
local_dir = cfg.upload_parts_dir / upload_id
local_dir.mkdir(parents=True, exist_ok=False)
return store.create_upload(
item_id=upload_id,
owner_user_id=request.state.user.get("id"),
filename=filename,
content_type=payload.content_type,
total_size=payload.total_size,
chunk_size=payload.chunk_size,
total_parts=math.ceil(payload.total_size / payload.chunk_size),
checksum_expected=payload.checksum.lower(),
local_dir=local_dir,
)
@app.get("/api/uploads/{upload_id}")
def get_upload(upload_id: str, request: Request) -> dict[str, Any]:
try:
upload = store.get_upload(upload_id)
_check_upload_owner(upload, request.state.user)
return upload
except KeyError as exc:
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
@app.put("/api/uploads/{upload_id}/parts/{part_number}")
async def upload_part(upload_id: str, part_number: int, request: Request) -> dict[str, Any]:
try:
upload = store.get_upload(upload_id)
_check_upload_owner(upload, request.state.user)
except KeyError as exc:
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
if upload["status"] != "uploading":
raise HTTPException(status_code=409, detail="上传任务已进入终态")
if not 1 <= part_number <= upload["total_parts"]:
raise HTTPException(status_code=400, detail="分片序号超出范围")
expected_size = upload["chunk_size"]
if part_number == upload["total_parts"]:
expected_size = upload["total_size"] - upload["chunk_size"] * (upload["total_parts"] - 1)
target = Path(upload["local_dir"]) / f"part-{part_number:08d}"
temporary = target.with_suffix(".tmp")
digest = hashlib.sha256()
written = 0
try:
with temporary.open("wb") as handle:
async for chunk in request.stream():
written += len(chunk)
if written > expected_size:
raise HTTPException(status_code=413, detail="分片大小超过声明值")
handle.write(chunk)
digest.update(chunk)
handle.flush()
os.fsync(handle.fileno())
if written != expected_size:
raise HTTPException(status_code=400, detail=f"分片大小不匹配,期望 {expected_size} 字节,实际 {written} 字节")
temporary.replace(target)
finally:
if temporary.exists():
temporary.unlink()
result = store.record_upload_part(upload_id, part_number)
return {"upload": result, "part_number": part_number, "size_bytes": written, "checksum": digest.hexdigest()}
@app.post("/api/uploads/{upload_id}/complete")
def complete_upload(upload_id: str, request: Request) -> dict[str, Any]:
try:
upload = store.get_upload(upload_id)
_check_upload_owner(upload, request.state.user)
except KeyError as exc:
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
if upload["status"] == "completed":
return {"upload": upload, "dataset": _dataset_for_storage_uri(store, upload["storage_uri"])}
missing = sorted(set(range(1, upload["total_parts"] + 1)) - set(upload["received_parts"]))
if missing:
raise HTTPException(status_code=409, detail={"message": "仍有分片未上传", "missing_parts": missing[:100]})
local_dir = Path(upload["local_dir"])
assembled = local_dir / upload["filename"]
digest = hashlib.sha256()
with assembled.open("wb") as output:
for part_number in range(1, upload["total_parts"] + 1):
part = local_dir / f"part-{part_number:08d}"
with part.open("rb") as handle:
while chunk := handle.read(8 * 1024 * 1024):
output.write(chunk)
digest.update(chunk)
output.flush()
os.fsync(output.fileno())
checksum = digest.hexdigest()
if assembled.stat().st_size != upload["total_size"] or (upload["checksum_expected"] and checksum != upload["checksum_expected"]):
store.fail_upload(upload_id, "assembled file checksum or size mismatch")
raise HTTPException(status_code=422, detail="完整文件校验失败,请续传错误分片后重试")
try:
uploaded = storage.upload_file(
assembled,
area="uploads",
relative_path=f"{upload_id}/{upload['filename']}",
metadata={"sha256": checksum, "upload-id": upload_id},
)
dataset_id = secrets.token_hex(16)
dataset_dir = cfg.datasets_dir / dataset_id
dataset_dir.mkdir(parents=True, exist_ok=False)
if assembled.suffix.lower() == ".zip":
with zipfile.ZipFile(assembled) as archive:
_safe_extract(archive, dataset_dir)
else:
shutil.copy2(assembled, dataset_dir / upload["filename"])
scan = scan_dataset(dataset_dir)
manifest_path = dataset_dir / "dataset_manifest.json"
manifest = {
"schema": "real2sim.dataset_manifest.v2",
"id": dataset_id,
"name": Path(upload["filename"]).stem,
"source_type": "browser_upload",
"storage_uri": uploaded["storage_uri"],
"checksum": checksum,
"version": checksum[:12],
"scan": scan,
}
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
storage.upload_file(manifest_path, area="datasets", relative_path=f"{dataset_id}/dataset_manifest.json", metadata={"sha256": sha256_file(manifest_path)})
dataset = store.create_dataset(
item_id=dataset_id,
name=manifest["name"],
path=dataset_dir,
scan=scan,
source_type="browser_upload",
source_uri=uploaded["storage_uri"],
version=manifest["version"],
manifest_path=manifest_path,
storage_uri=uploaded["storage_uri"],
checksum=checksum,
cache_state="local",
)
completed = store.complete_upload(upload_id, checksum_actual=checksum, storage_uri=uploaded["storage_uri"])
return {"upload": completed, "dataset": _with_view_model(dataset, "dataset")}
except (StorageUnavailable, StorageBoundaryError) as exc:
store.fail_upload(upload_id, str(exc))
raise HTTPException(status_code=503, detail=str(exc)) from exc
@app.delete("/api/uploads/{upload_id}")
def delete_upload(upload_id: str, request: Request) -> dict[str, bool]:
try:
upload = store.get_upload(upload_id)
_check_upload_owner(upload, request.state.user)
shutil.rmtree(Path(upload["local_dir"]), ignore_errors=True)
store.delete_upload(upload_id)
return {"ok": True}
except KeyError as exc:
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
@app.get("/api/storage/objects")
def list_storage_objects(area: str = "datasets", prefix: str = "", max_keys: int = 200) -> dict[str, Any]:
try:
return {"bucket": storage.bucket, "prefix": storage.prefix, "objects": storage.list_objects(area=area, relative_prefix=prefix, max_keys=max_keys)}
except (StorageUnavailable, StorageBoundaryError) as exc:
raise HTTPException(status_code=503 if isinstance(exc, StorageUnavailable) else 400, detail=str(exc)) from exc
@app.post("/api/datasets/import-bos")
def import_bos_dataset(payload: ImportBosRequest) -> dict[str, Any]:
return _queue_task("bos_import", {"prefix": payload.prefix, "name": payload.name or Path(payload.prefix.rstrip("/")).name, "max_objects": payload.max_objects})
@app.get("/api/datasets")
def list_datasets() -> list[dict[str, Any]]:
@ -138,6 +541,8 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
@app.post("/api/datasets/upload")
def upload_dataset(file: UploadFile = File(...)) -> dict[str, Any]:
if cfg.production_mode:
raise HTTPException(status_code=410, detail="生产模式请使用 /api/uploads 分片上传,旧上传接口已禁用")
target = cfg.uploads_dir / Path(file.filename or "upload.bin").name
with target.open("wb") as handle:
shutil.copyfileobj(file.file, handle)
@ -212,8 +617,15 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
raise HTTPException(status_code=404, detail="asset file not found")
if not is_path_allowed(asset_path, cfg):
raise HTTPException(status_code=403, detail="asset path is outside configured import whitelist")
manifest_path = _scene_manifest_path_for_write(scene)
manifest = _read_manifest_or_default(scene, manifest_path)
revision_id = secrets.token_hex(16)
revision_dir = cfg.scenes_dir / "revisions" / revision_id
source_scene_path = Path(scene["path"])
source_package = source_scene_path if source_scene_path.is_dir() else source_scene_path.parent
_copy_scene_revision(source_package, revision_dir)
revision_scene_path = revision_dir / source_scene_path.name if source_scene_path.is_file() else revision_dir
source_manifest_path = _scene_manifest_path_for_write(scene)
manifest_path = revision_dir / source_manifest_path.name
manifest = _read_manifest_or_default(scene, source_manifest_path)
artifact = store.add_artifact(task_id=None, name=payload.name or asset_path.name, path=asset_path, kind=(payload.kind or _scene_asset_kind(asset_path)))
asset = _scene_asset_entry(
asset_path,
@ -226,12 +638,28 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and str(item.get("path") or "") == str(asset_path))]
manifest["assets"] = [asset] + assets
manifest["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
manifest["revision"] = {"id": revision_id, "parent_scene_id": scene_id, "reason": "asset_added"}
manifest["asset_editing"] = {"supports_added_assets": True, "last_added_asset": asset["name"]}
manifest_path.parent.mkdir(parents=True, exist_ok=True)
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset"]))
store.update_scene(scene_id, tags=tags)
return {"ok": True, "scene": store.get_scene(scene_id), "asset": asset, "manifest_path": str(manifest_path)}
tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset", "scene_revision"]))
metadata = dict(scene.get("metadata") or {})
metadata.update({"parent_scene_id": scene_id, "revision_reason": "asset_added", "package": str(revision_dir)})
revision = store.create_scene(
item_id=revision_id,
name=f"{scene['name']} · 资产修订",
path=revision_scene_path,
metadata=metadata,
source_type="scene_revision",
source_uri=scene.get("storage_uri") or scene.get("source_uri") or str(source_scene_path),
version=_next_revision_label(str(scene.get("version") or "")),
parent_task_id=scene.get("parent_task_id"),
manifest_path=manifest_path,
metrics=scene.get("metrics") or {},
tags=tags,
cache_state="local",
)
return {"ok": True, "scene": revision, "parent_scene_id": scene_id, "asset": asset, "manifest_path": str(manifest_path)}
@app.patch("/api/scenes/{scene_id}")
def update_scene(scene_id: str, payload: UpdateEntityRequest) -> dict[str, Any]:
@ -277,9 +705,9 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
def _queue_task(task_type: str, params: dict[str, Any]) -> dict[str, Any]:
payload = CreateTaskRequest(type=task_type, params=params)
_validate_task_payload(payload)
_validate_task_payload(payload, cfg)
_validate_task_resources(store, payload)
if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "simulator_validation"}:
if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "simulator_validation"}:
raise ValueError(f"unknown task type: {payload.type}")
run_dir = cfg.runs_dir / payload.type / _next_run_name(cfg.runs_dir / payload.type, payload.type)
return store.create_task(task_type=payload.type, params=payload.params, run_dir=run_dir)
@ -376,6 +804,22 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
def list_tasks() -> list[dict[str, Any]]:
return [_with_task_view_model(item) for item in store.list_tasks()]
@app.get("/api/events/tasks")
async def task_events(request: Request):
async def stream():
previous = ""
while not await request.is_disconnected():
tasks = [_with_task_view_model(item) for item in store.list_tasks()]
fingerprint = hashlib.sha256(json.dumps([(item["id"], item["status"], item["progress"], item["stage"], item["message"], item["updated_at"]) for item in tasks], ensure_ascii=False).encode("utf-8")).hexdigest()
if fingerprint != previous:
yield f"event: tasks\ndata: {json.dumps(tasks, ensure_ascii=False)}\n\n"
previous = fingerprint
else:
yield ": keepalive\n\n"
await asyncio.sleep(1.5)
return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
@app.post("/api/tasks")
def create_task(payload: CreateTaskRequest) -> dict[str, Any]:
try:
@ -493,6 +937,29 @@ def _scene_manifest_path_for_write(scene: dict[str, Any]) -> Path:
return package_dir / "scene_manifest.json"
def _copy_scene_revision(source: Path, target: Path) -> None:
if not source.is_dir():
raise FileNotFoundError(source)
def link_or_copy(src: str, dst: str) -> str:
try:
os.link(src, dst)
return dst
except OSError:
return shutil.copy2(src, dst)
shutil.copytree(source, target, copy_function=link_or_copy)
def _next_revision_label(current: str) -> str:
import re
match = re.search(r"(?:^|[-_.])r(\d+)$", current, re.IGNORECASE)
number = int(match.group(1)) + 1 if match else 2
base = re.sub(r"(?:[-_.])r\d+$", "", current, flags=re.IGNORECASE).strip("-_.") or "v1"
return f"{base}-r{number}"
def _read_manifest_or_default(scene: dict[str, Any], manifest_path: Path) -> dict[str, Any]:
if manifest_path.is_file():
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
@ -735,8 +1202,15 @@ def _task_type_label(task_type: str) -> str:
}.get(task_type, task_type)
def _validate_task_payload(payload: CreateTaskRequest) -> None:
def _validate_task_payload(payload: CreateTaskRequest, cfg: PlatformConfig) -> None:
params = payload.params
if cfg.production_mode and bool(params.get("dry_run")):
raise HTTPException(status_code=400, detail="生产模式禁止 dry-run 任务")
if payload.type == "bos_import":
if not str(params.get("prefix") or "").strip():
raise HTTPException(status_code=400, detail="bos_import requires prefix")
if not 1 <= int(params.get("max_objects", 1000)) <= 1000:
raise HTTPException(status_code=400, detail="max_objects must be between 1 and 1000")
if payload.type == "hf_import":
repo_type = params.get("repo_type", "dataset")
if repo_type != "dataset":
@ -778,8 +1252,9 @@ def _validate_task_payload(payload: CreateTaskRequest) -> None:
if payload.type == "expert_collection":
if not (params.get("scene_id") or params.get("scene_path")):
raise HTTPException(status_code=400, detail="expert_collection requires scene_id or scene_path")
if params.get("policy_preset", "pick_and_place_v1") != "pick_and_place_v1":
raise HTTPException(status_code=400, detail="当前仅支持抓取放置策略;导航和精细操作策略尚未接入执行器")
supported_policy = "rm65_pick_place_6d_ik" if cfg.production_mode else "pick_and_place_v1"
if params.get("policy_preset", supported_policy) != supported_policy:
raise HTTPException(status_code=400, detail=f"当前运行模式仅允许真实控制器 {supported_policy}")
if int(params.get("workers", 1)) != 1:
raise HTTPException(status_code=400, detail="当前专家采集执行器仅支持 workers=1")
if int(params.get("episodes", 4)) <= 0 or int(params.get("steps", 24)) <= 0:
@ -787,8 +1262,9 @@ def _validate_task_payload(payload: CreateTaskRequest) -> None:
if payload.type == "training":
if not (params.get("dataset_id") or params.get("dataset_path")):
raise HTTPException(status_code=400, detail="training requires dataset_id or dataset_path")
if params.get("recipe", "bc_v2") not in {"bc_v2", "bc_fast"}:
raise HTTPException(status_code=400, detail="当前仅支持行为克隆 v2 和快速验证;扩散策略尚未接入训练器")
allowed_recipes = {"discoverse_act"} if cfg.production_mode else {"bc_v2", "bc_fast", "discoverse_act"}
if params.get("recipe", "discoverse_act" if cfg.production_mode else "bc_v2") not in allowed_recipes:
raise HTTPException(status_code=400, detail="生产模式只允许 DISCOVERSE ACT;历史岭回归配方仅可查看")
if int(params.get("epochs", 100)) <= 0:
raise HTTPException(status_code=400, detail="epochs must be > 0")
if payload.type == "inference":
@ -815,13 +1291,133 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) -
raise HTTPException(status_code=400, detail="训练只能使用专家采集产生的 rollout 数据集")
if payload.type == "inference":
if params.get("model_id"):
store.get_model(str(params["model_id"]))
model = store.get_model(str(params["model_id"]))
recipe = str((model.get("metadata") or {}).get("training", {}).get("recipe") or (model.get("metadata") or {}).get("recipe") or "")
if params.get("recipe") == "discoverse_act" and recipe and recipe != "discoverse_act":
raise HTTPException(status_code=400, detail="所选模型不是 DISCOVERSE ACT 模型")
if params.get("scene_id"):
store.get_scene(str(params["scene_id"]))
except KeyError as exc:
raise HTTPException(status_code=400, detail=f"任务引用的资源不存在: {exc.args[0]}") from exc
def _active_admin_count(store: PlatformStore) -> int:
return sum(1 for user in store.list_users() if user["role"] == "admin" and not user["disabled"])
def _check_upload_owner(upload: dict[str, Any], user: dict[str, Any]) -> None:
if user.get("role") != "admin" and upload.get("owner_user_id") != user.get("id"):
raise HTTPException(status_code=403, detail="无权访问其他用户的上传任务")
def _dataset_for_storage_uri(store: PlatformStore, storage_uri: str) -> dict[str, Any] | None:
return next((item for item in store.list_datasets() if item.get("storage_uri") == storage_uri), None)
def _sqlite_journal_mode(store: PlatformStore) -> str:
with store._connect() as db:
return str(db.execute("pragma journal_mode").fetchone()[0]).lower()
def _capabilities(cfg: PlatformConfig, storage: BosStorage, store: PlatformStore | None = None) -> dict[str, Any]:
anysplat_commit = _git_commit_full(cfg.anysplat_root)
discoverse_commit = _git_commit_full(cfg.discoverse_root)
act_root = cfg.discoverse_root / "policies" / "act"
act_commit = _git_commit_full(act_root)
ffmpeg_ready = shutil.which("ffmpeg") is not None
colmap_ready = cfg.colmap_binary.is_file() or shutil.which("colmap") is not None
mujoco_ready = _module_available("mujoco")
motrixsim_ready = _module_available("motrixsim") or bool(shutil.which("motrixsim"))
torch_ready = _module_available("torch")
rm65_controller = Path(__file__).with_name("rm65_controller.py")
local_runtime = {
"ffmpeg": ffmpeg_ready,
"colmap": colmap_ready,
"anysplat": cfg.anysplat_root.is_dir() and cfg.anysplat_python.is_file() and anysplat_commit == cfg.anysplat_commit,
"mujoco": mujoco_ready,
"motrixsim": motrixsim_ready,
"discoverse_act": cfg.discoverse_root.is_dir() and act_root.is_dir() and discoverse_commit == cfg.discoverse_commit and act_commit == cfg.act_commit and torch_ready,
"rm65_controller": rm65_controller.is_file() and mujoco_ready,
}
worker_status: dict[str, Any] = {"connected": cfg.worker_mode != "api", "age_seconds": 0 if cfg.worker_mode != "api" else None}
if store is not None and cfg.worker_mode == "api":
heartbeats = store.list_worker_heartbeats()
latest = heartbeats[0] if heartbeats else None
age = time.time() - float(latest["updated_at"]) if latest else None
connected = bool(latest and age is not None and age < 30)
worker_status = {"connected": connected, "age_seconds": age, "worker_id": latest.get("worker_id") if latest else None}
if connected:
checks = latest["payload"].get("checks", {})
local_runtime.update(
ffmpeg=bool(checks.get("ffmpeg")),
colmap=bool(checks.get("colmap")),
anysplat=all(bool(checks.get(name)) for name in ("anysplat_root", "anysplat_python", "anysplat_pinned")),
mujoco=bool(checks.get("mujoco_python")),
motrixsim=bool(checks.get("motrixsim_python")),
discoverse_act=all(bool(checks.get(name)) for name in ("discoverse_act", "torch_cuda", "act_import")),
rm65_controller=bool(checks.get("rm65_controller")) and bool(checks.get("mujoco_python")),
)
else:
for name in local_runtime:
local_runtime[name] = False
return {
"schema": "real2sim.capabilities.v1",
"production_mode": cfg.production_mode,
"auth_enabled": cfg.auth_enabled,
"worker_mode": cfg.worker_mode,
"worker": worker_status,
"storage": {
"bos": {
"configured": storage.configured,
"bucket": storage.bucket,
"prefix": storage.prefix,
"resumable_upload": True,
"sha256": True,
}
},
"runtimes": {
"ffmpeg": {"ready": local_runtime["ffmpeg"]},
"colmap": {"ready": local_runtime["colmap"]},
"anysplat": {
"ready": local_runtime["anysplat"],
"commit": anysplat_commit,
"required_commit": cfg.anysplat_commit,
"model_revision": cfg.anysplat_model_revision,
"vggt_model_revision": cfg.vggt_model_revision,
},
"mujoco": {"ready": local_runtime["mujoco"]},
"motrixsim": {"ready": local_runtime["motrixsim"]},
"discoverse_act": {
"ready": local_runtime["discoverse_act"],
"discoverse_commit": discoverse_commit,
"required_discoverse_commit": cfg.discoverse_commit,
"act_commit": act_commit,
"required_act_commit": cfg.act_commit,
},
"rm65_controller": {"ready": local_runtime["rm65_controller"]},
"spark": {"ready": SPARK_VENDOR_PATH.is_file()},
},
}
def _git_commit_full(root: Path) -> str:
if not root.is_dir():
return ""
try:
return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=3).strip()
except Exception:
return ""
def _module_available(name: str) -> bool:
try:
import importlib.util
return importlib.util.find_spec(name) is not None
except Exception:
return False
def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]:
return {
"gs_playground_root": str(cfg.gs_playground_root),
@ -831,6 +1427,11 @@ def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]:
"gsplat_python": str(cfg.gsplat_python),
"cuda_home": str(cfg.cuda_home),
"runtime_env_script": str(cfg.runtime_env_script),
"discoverse_commit": cfg.discoverse_commit,
"act_commit": cfg.act_commit,
"anysplat_commit": cfg.anysplat_commit,
"anysplat_model_revision": cfg.anysplat_model_revision,
"vggt_model_revision": cfg.vggt_model_revision,
}

View File

@ -1,4 +1,4 @@
const STATIC_VERSION = "20260710-motrix-runtime-v7";
const STATIC_VERSION = "20260710-poc-product-v1";
const state = {
datasets: [],
@ -11,7 +11,13 @@ const state = {
editing: null,
assetLibrary: [],
refreshTimer: null,
taskEvents: null,
selections: {},
user: null,
capabilities: null,
recipes: null,
initialized: false,
pendingPassword: "",
};
const $ = (id) => document.getElementById(id);
@ -25,13 +31,97 @@ const taskScopes = {
};
async function api(path, options = {}) {
const response = await fetch(path, options);
const response = await fetch(path, { credentials: "same-origin", ...options });
const type = response.headers.get("content-type") || "";
const payload = type.includes("application/json") ? await response.json() : await response.text();
if (!response.ok) throw new Error(payload?.detail || payload?.error || response.statusText);
if (!response.ok) {
const detail = typeof payload?.detail === "object" ? payload.detail.message || JSON.stringify(payload.detail) : payload?.detail || payload?.error || response.statusText;
const error = new Error(detail);
error.status = response.status;
error.payload = payload;
throw error;
}
return payload;
}
function showAuth(mode = "login") {
$("appShell").hidden = true;
$("authShell").hidden = false;
$("loginForm").hidden = mode !== "login";
$("passwordForm").hidden = mode !== "password";
if (mode === "password" && state.pendingPassword) $("currentPassword").value = state.pendingPassword;
}
function showApp() {
$("authShell").hidden = true;
$("appShell").hidden = false;
setText("currentUserLabel", `${state.user?.username || "本地用户"} · ${roleLabel(state.user?.role)}`);
const usersButton = $("openUsers");
if (usersButton) usersButton.hidden = state.user?.role !== "admin";
}
async function bootstrap() {
state.capabilities = await api("/api/capabilities");
let authState;
try {
authState = await api("/api/auth/me");
} catch (error) {
if (error.status === 401) return showAuth("login");
throw error;
}
state.user = authState.user;
if (state.user?.must_change_password) return showAuth("password");
state.recipes = await api("/api/recipes");
applyRuntimeOptions();
showApp();
await refresh();
startTaskEvents();
clearInterval(state.refreshTimer);
state.refreshTimer = setInterval(() => refresh({ quiet: true }).catch((error) => console.warn(error)), 15000);
state.initialized = true;
}
function startTaskEvents() {
state.taskEvents?.close();
state.taskEvents = new EventSource("/api/events/tasks");
state.taskEvents.addEventListener("tasks", (event) => {
const previousTerminal = new Set(state.tasks.filter((task) => terminalStatus(task.status)).map((task) => task.id));
state.tasks = JSON.parse(event.data);
renderScopedTaskLists();
renderOverview();
if (state.selectedTaskId && $("taskDrawer")?.classList.contains("open")) showTask(state.selectedTaskId, { preservePreview: true }).catch(console.warn);
if (state.tasks.some((task) => terminalStatus(task.status) && !previousTerminal.has(task.id))) refresh({ quiet: true }).catch(console.warn);
});
state.taskEvents.onerror = () => console.warn("任务事件连接暂时中断,浏览器将自动重连");
}
function terminalStatus(status) {
return ["succeeded", "failed", "failed_recovered", "canceled"].includes(status);
}
function roleLabel(role) {
return { admin: "管理员", engineer: "工程师", viewer: "只读" }[role] || "本地模式";
}
function applyRuntimeOptions() {
const collection = state.recipes?.collection || [];
const training = state.recipes?.training || [];
fillRecipeSelect("policyPreset", collection, "暂无可用真实控制器");
fillRecipeSelect("trainRecipe", training.filter((item) => !item.read_only), "暂无可用训练配方");
}
function fillRecipeSelect(id, items, emptyLabel) {
const select = $(id);
if (!select) return;
const enabled = items.find((item) => item.enabled);
const unavailableReason = items.find((item) => !item.enabled)?.reason;
select.innerHTML = enabled
? items.map((item) => `<option value="${escapeAttr(item.id)}" ${item.enabled ? "" : "disabled"}>${escapeHtml(item.name)}${item.enabled ? "" : ` · ${item.reason || "不可用"}`}</option>`).join("")
: `<option value="">${escapeHtml(unavailableReason || emptyLabel)}</option>${items.map((item) => `<option value="${escapeAttr(item.id)}" disabled>${escapeHtml(item.name)} · ${escapeHtml(item.reason || "不可用")}</option>`).join("")}`;
select.value = enabled?.id || "";
select.disabled = !enabled;
}
async function refresh({ quiet = false } = {}) {
try {
const [datasets, scenes, models, tasks] = await Promise.all([
@ -77,13 +167,15 @@ function renderAll() {
}
function updateActionAvailability() {
setActionAvailability("createMediaPrepare", state.datasets.length > 0, "请先新增数据集");
setActionAvailability("createPreflightTask", preparedDatasets().length > 0, "请先准备帧数据");
setActionAvailability("createSceneTask", preparedDatasets().length > 0, "请先准备帧数据");
setActionAvailability("createCollectTask", state.scenes.length > 0, "请先完成场景重建");
setActionAvailability("createTrainTask", collectionDatasets().length > 0, "请先完成专家采集");
setActionAvailability("createInferTask", state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型");
setActionAvailability("addSceneAsset", state.scenes.length > 0, "请先完成场景重建");
const writable = state.user?.role !== "viewer";
document.body.classList.toggle("viewer-mode", !writable);
setActionAvailability("createMediaPrepare", writable && state.datasets.length > 0, writable ? "请先新增数据集" : "只读用户不能创建任务");
setActionAvailability("createPreflightTask", writable && preparedDatasets().length > 0, "请先准备帧数据");
setActionAvailability("createSceneTask", writable && preparedDatasets().length > 0, "请先准备帧数据");
setActionAvailability("createCollectTask", writable && state.scenes.length > 0 && Boolean($("policyPreset")?.value), "请先选择兼容真实控制器的场景");
setActionAvailability("createTrainTask", writable && collectionDatasets().length > 0 && Boolean($("trainRecipe")?.value), "请先完成真实专家采集并确认 ACT 运行时");
setActionAvailability("createInferTask", writable && state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型");
setActionAvailability("addSceneAsset", writable && state.scenes.length > 0, "请先完成场景重建");
}
function setActionAvailability(id, enabled, reason) {
@ -293,14 +385,24 @@ function renderTrainingChart() {
const best = state.models[0];
const score = best ? modelScore(best) : 0;
const loss = best?.metrics?.loss ?? best?.metrics?.val_loss ?? null;
if (!best) {
setHtml("trainingChart", emptyState("完成一次真实训练后,这里会显示损失曲线和验证指标。"));
return;
}
const history = [best?.metrics?.loss_history, best?.metrics?.train_loss_history, best?.metrics?.loss_curve]
.find((value) => Array.isArray(value) && value.length > 1) || [];
const finiteHistory = history.map(Number).filter(Number.isFinite).slice(-48);
const maxLoss = finiteHistory.length ? Math.max(...finiteHistory) : 0;
const minLoss = finiteHistory.length ? Math.min(...finiteHistory) : 0;
const range = Math.max(maxLoss - minLoss, Number.EPSILON);
setHtml("trainingChart", `
<div class="chart-kpis">
<div><span>成功率</span><strong>${best ? `${Math.round(score)}%` : "--"}</strong></div>
<div><span>成功率</span><strong>${Number.isFinite(score) ? `${Math.round(score)}%` : "--"}</strong></div>
<div><span>验证损失</span><strong>${loss !== null ? Number(loss).toFixed(3) : "--"}</strong></div>
</div>
<div class="line-chart" aria-label="训练曲线">
${[35, 48, 54, 63, Math.max(38, score || 72), Math.max(44, Math.min(96, (score || 76) + 8))].map((height) => `<span style="height:${height}%"></span>`).join("")}
</div>
${finiteHistory.length > 1
? `<div class="line-chart" aria-label="真实训练损失曲线">${finiteHistory.map((value) => `<span style="height:${Math.max(8, 100 - ((value - minLoss) / range) * 88)}%" title="${escapeAttr(value.toFixed(6))}"></span>`).join("")}</div>`
: `<div class="inline-empty">该模型没有可绘制的损失历史;可在模型卡中查看最终指标。</div>`}
`);
}
@ -653,7 +755,7 @@ function showBulkResult(title, result) {
function openImportDialog(type) {
state.currentImport = type;
const title = { hf: "导入 HuggingFace 数据", upload: "上传数据文件", local: "导入本地目录" }[type] || "新增数据";
const title = { hf: "导入 HuggingFace 数据", upload: "上传数据文件", bos: "从 BOS 导入", local: "导入本地目录" }[type] || "新增数据";
setText("importTitle", title);
setText("importSubtitle", "主流程只展示资源选择;技术参数集中在这里配置。");
if (type === "hf") {
@ -672,7 +774,18 @@ function openImportDialog(type) {
</details>
`);
} else if (type === "upload") {
setHtml("importFields", `<label class="span-2">选择文件<input id="uploadFile" type="file"></label>`);
setHtml("importFields", `
<label class="span-2">选择文件<input id="uploadFile" type="file"></label>
<div id="uploadProgress" class="upload-progress span-2"><span></span><strong>等待选择文件</strong></div>
<p class="field-note span-2">上传会按 8MB 分片写入 BOS,刷新或网络中断后重新选择同一文件即可续传。</p>
`);
} else if (type === "bos") {
setHtml("importFields", `
<label>BOS 区域<select id="bosArea"><option value="datasets">数据集</option><option value="uploads">上传文件</option></select></label>
<label>显示名称<input id="bosDatasetName" placeholder="例如 Playroom 原始数据"></label>
<label class="span-2">授权前缀<input id="bosPrefix" placeholder="team/playroom/2026-07"></label>
<p class="field-note span-2">只能访问 real2sim-poc/ 下的授权区域,无法列举或删除其他项目数据。</p>
`);
} else {
setHtml("importFields", `
<label class="span-2">白名单目录<input id="datasetPath" placeholder="选择或粘贴配置白名单内目录"></label>
@ -695,10 +808,24 @@ async function submitImport(event) {
} else if (state.currentImport === "upload") {
const file = $("uploadFile")?.files?.[0];
if (!file) return toast("请先选择文件。");
const body = new FormData();
body.append("file", file);
await api("/api/datasets/upload", { method: "POST", body });
toast("文件已上传并登记为数据集");
if (state.capabilities?.storage?.bos?.configured) {
await uploadFileResumable(file);
} else {
const body = new FormData();
body.append("file", file);
await api("/api/datasets/upload", { method: "POST", body });
}
toast("文件已校验并登记为数据集");
await refresh({ quiet: true });
} else if (state.currentImport === "bos") {
const prefix = $("bosPrefix")?.value.trim();
if (!prefix) return toast("请填写授权前缀。");
await api("/api/datasets/import-bos", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ prefix: `${$("bosArea")?.value || "datasets"}/${prefix}`, name: $("bosDatasetName")?.value.trim() || undefined }),
});
toast("BOS 导入任务已创建");
await refresh({ quiet: true });
} else {
const path = $("datasetPath")?.value.trim();
@ -714,6 +841,50 @@ async function submitImport(event) {
$("importDialog")?.close();
}
async function uploadFileResumable(file) {
const chunkSize = 8 * 1024 * 1024;
const resumeKey = `real2sim-upload:${file.name}:${file.size}:${file.lastModified}`;
let upload = null;
const savedId = localStorage.getItem(resumeKey);
if (savedId) {
try {
upload = await api(`/api/uploads/${savedId}`);
if (upload.filename !== file.name || upload.total_size !== file.size || upload.status !== "uploading") upload = null;
} catch (_error) {
upload = null;
}
}
if (!upload) {
upload = await api("/api/uploads", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ filename: file.name, content_type: file.type || "application/octet-stream", total_size: file.size, chunk_size: chunkSize }),
});
localStorage.setItem(resumeKey, upload.id);
}
const received = new Set(upload.received_parts || []);
for (let part = 1; part <= upload.total_parts; part += 1) {
if (received.has(part)) continue;
const start = (part - 1) * upload.chunk_size;
const end = Math.min(file.size, start + upload.chunk_size);
await api(`/api/uploads/${upload.id}/parts/${part}`, { method: "PUT", headers: { "Content-Type": "application/octet-stream" }, body: file.slice(start, end) });
received.add(part);
renderUploadProgress(received.size / upload.total_parts, `已上传 ${received.size}/${upload.total_parts} 个分片`);
}
renderUploadProgress(0.98, "正在校验 SHA-256 并写入 BOS");
const completed = await api(`/api/uploads/${upload.id}/complete`, { method: "POST" });
localStorage.removeItem(resumeKey);
renderUploadProgress(1, `上传完成 · ${completed.upload.checksum_actual.slice(0, 12)}`);
return completed;
}
function renderUploadProgress(value, label) {
const node = $("uploadProgress");
if (!node) return;
node.querySelector("span").style.width = `${Math.max(0, Math.min(100, value * 100))}%`;
node.querySelector("strong").textContent = label;
}
function openViewer(sceneId) {
state.activeSceneId = sceneId;
const assetScene = $("assetScene");
@ -726,9 +897,19 @@ function openViewer(sceneId) {
}
function openViewerFullscreen(url) {
const viewerUrl = url || $("openSceneViewerFull")?.dataset.viewerUrl || $("scenePreview")?.getAttribute("src");
if (!viewerUrl) return toast("请先选择一个场景。");
window.open(viewerUrl, "_blank", "noopener,noreferrer");
const panel = document.querySelector(".viewer-panel");
const button = $("openSceneViewerFull");
const iframe = $("scenePreview");
const viewerUrl = url || button?.dataset.viewerUrl || iframe?.getAttribute("src");
if (!panel || !viewerUrl) return toast("请先选择一个场景。");
const maximizing = !panel.classList.contains("is-maximized");
panel.classList.toggle("is-maximized", maximizing);
document.body.classList.toggle("viewer-maximized", maximizing);
if (button) {
button.textContent = maximizing ? "退出最大化" : "最大化预览";
button.setAttribute("aria-pressed", String(maximizing));
}
if (iframe && maximizing && !iframe.getAttribute("src")) iframe.src = viewerUrl;
}
function openAssetEditor(sceneId) {
@ -822,7 +1003,7 @@ function reconstructionOptions() {
scene_name: $("sceneName")?.value.trim() || undefined,
renderer: $("rendererSelect")?.value || "auto",
simulators: simulatorValues(),
dry_run: $("dryRunScene")?.checked || false,
dry_run: false,
auto_prepare: $("autoPrepareScene")?.checked || false,
allow_low_quality: $("allowLowQualityScene")?.checked || false,
min_frames: Number($("sceneMinFrames")?.value || 30),
@ -1267,7 +1448,100 @@ function toast(message) {
node._timer = setTimeout(() => node.classList.remove("show"), 2600);
}
async function submitLogin(event) {
event.preventDefault();
setText("loginError", "");
const username = $("loginUsername").value.trim();
const password = $("loginPassword").value;
try {
const result = await api("/api/auth/login", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ username, password }),
});
state.user = result.user;
state.pendingPassword = password;
if (state.user?.must_change_password) {
showAuth("password");
return;
}
state.pendingPassword = "";
await bootstrap();
} catch (error) {
setText("loginError", error.message || "登录失败");
}
}
async function submitPasswordChange(event) {
event.preventDefault();
setText("passwordError", "");
const currentPassword = $("currentPassword").value;
const newPassword = $("newPassword").value;
if (newPassword !== $("confirmPassword").value) return setText("passwordError", "两次输入的新密码不一致");
try {
await api("/api/auth/change-password", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ current_password: currentPassword, new_password: newPassword }),
});
state.pendingPassword = "";
$("loginPassword").value = "";
showAuth("login");
setText("loginError", "密码已更新,请使用新密码登录");
} catch (error) {
setText("passwordError", error.message || "密码更新失败");
}
}
async function logout() {
try {
await api("/api/auth/logout", { method: "POST" });
} finally {
state.taskEvents?.close();
clearInterval(state.refreshTimer);
state.user = null;
state.initialized = false;
showAuth("login");
}
}
async function openUserManager() {
await renderUsers();
$("usersDialog")?.showModal();
}
async function renderUsers() {
const users = await api("/api/users");
setHtml("userList", users.map((user) => `
<div class="user-row">
<div><strong>${escapeHtml(user.username)}</strong><span>${escapeHtml(roleLabel(user.role))} · ${user.disabled ? "已禁用" : user.must_change_password ? "待首次改密" : "可用"}</span></div>
<div class="row-actions">
<button data-user-toggle="${escapeAttr(user.id)}" data-user-disabled="${user.disabled}">${user.disabled ? "启用" : "禁用"}</button>
<button data-user-reset="${escapeAttr(user.id)}">重置密码</button>
<button class="danger-button" data-user-delete="${escapeAttr(user.id)}">删除</button>
</div>
</div>
`).join("") || emptyState("暂无团队用户。"));
}
async function createUser(event) {
event.preventDefault();
await api("/api/users", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ username: $("newUsername").value.trim(), password: $("newUserPassword").value, role: $("newUserRole").value }),
});
event.target.reset();
toast("用户已创建,首次登录必须修改密码");
await renderUsers();
}
function bindEvents() {
$("loginForm")?.addEventListener("submit", submitLogin);
$("passwordForm")?.addEventListener("submit", submitPasswordChange);
$("logoutButton")?.addEventListener("click", logout);
$("openUsers")?.addEventListener("click", openUserManager);
$("createUserForm")?.addEventListener("submit", createUser);
document.querySelectorAll(".tab").forEach((button) => button.addEventListener("click", () => switchTab(button.dataset.tab)));
$("closeDrawer")?.addEventListener("click", closeDrawer);
$("importForm")?.addEventListener("submit", submitImport);
@ -1323,13 +1597,13 @@ function bindEvents() {
if (!sceneId) return toast("请先选择一个场景。");
const payload = sceneAssetPayload();
if (!payload.path) return toast("请选择资产来源,或在高级入口填写白名单文件。");
await api(`/api/scenes/${sceneId}/assets`, {
const result = await api(`/api/scenes/${sceneId}/assets`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
state.activeSceneId = sceneId;
toast("资产已添加到场景清单");
state.activeSceneId = result.scene?.id || sceneId;
toast("资产已添加,并创建了新的场景修订版本");
await refresh({ quiet: true });
renderSceneViewerShell();
scrollIntoViewIfPossible("#assetRail");
@ -1358,7 +1632,15 @@ function bindEvents() {
const sceneId = $("inferScene")?.value || state.scenes[0]?.id;
if (!modelId) return toast("请先训练模型。");
if (!sceneId) return toast("请先完成场景重建。");
await createTask("inference", { model_id: modelId, scene_id: sceneId, steps: Number($("inferSteps")?.value || 80), workers: Number($("inferWorkers")?.value || 1) });
await createTask("inference", {
model_id: modelId,
scene_id: sceneId,
recipe: "discoverse_act",
episodes: Number($("inferEpisodes")?.value || 10),
steps: Number($("inferSteps")?.value || 240),
workers: Number($("inferWorkers")?.value || 1),
minimum_success_rate: 0.5,
});
});
document.body.addEventListener("change", (event) => {
@ -1378,6 +1660,37 @@ async function handleBodyClick(event) {
const closeDialog = event.target.closest("[data-close-dialog]");
if (closeDialog) $(closeDialog.dataset.closeDialog)?.close();
const userToggle = event.target.closest("[data-user-toggle]");
if (userToggle) {
await api(`/api/users/${userToggle.dataset.userToggle}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ disabled: userToggle.dataset.userDisabled !== "true" }),
});
await renderUsers();
}
const userReset = event.target.closest("[data-user-reset]");
if (userReset) {
const password = prompt("输入至少 12 位的新初始密码;用户下次登录仍需修改密码。") || "";
if (password) {
await api(`/api/users/${userReset.dataset.userReset}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ reset_password: password }),
});
toast("密码已重置");
await renderUsers();
}
}
const userDelete = event.target.closest("[data-user-delete]");
if (userDelete && confirm("删除用户会终止其会话,但不会删除业务资源。确认继续?")) {
await api(`/api/users/${userDelete.dataset.userDelete}`, { method: "DELETE" });
toast("用户已删除");
await renderUsers();
}
const openImport = event.target.closest("[data-open-import]");
if (openImport) openImportDialog(openImport.dataset.openImport);
@ -1482,9 +1795,20 @@ async function handleBodyClick(event) {
}
}
window.addEventListener("unhandledrejection", (event) => toast(event.reason?.message || String(event.reason || "操作失败")));
window.addEventListener("unhandledrejection", (event) => {
if (event.reason?.status === 401) showAuth("login");
if (event.reason?.status === 428) showAuth("password");
toast(event.reason?.message || String(event.reason || "操作失败"));
});
window.addEventListener("error", (event) => event.message && toast(event.message));
window.addEventListener("keydown", (event) => {
if (event.key === "Escape" && document.querySelector(".viewer-panel.is-maximized")) {
openViewerFullscreen();
}
});
bindEvents();
refresh().catch((error) => toast(error.message));
state.refreshTimer = setInterval(() => refresh({ quiet: true }).catch((error) => console.warn(error)), 4000);
bootstrap().catch((error) => {
toast(error.message || "平台初始化失败");
if (error.status === 401) showAuth("login");
});

View File

@ -5,10 +5,43 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Real2Sim</title>
<link rel="icon" href="data:,">
<link rel="stylesheet" href="/static/styles.css?v=20260710-motrix-runtime-v7">
<link rel="stylesheet" href="/static/styles.css?v=20260710-poc-product-v1">
</head>
<body>
<div class="app-shell">
<section id="authShell" class="auth-shell" aria-label="登录 Real2Sim">
<div class="auth-brand">
<strong>Real2Sim</strong>
<span>机器人从真实场景走向可训练智能体</span>
<div class="auth-flow" aria-hidden="true">
<i class="done"></i><b></b><i></i><b></b><i></i>
</div>
<p>数据、场景、采集、训练与推理使用同一条可追踪链路。</p>
</div>
<form id="loginForm" class="auth-card">
<div>
<h1>登录平台</h1>
<p>使用管理员分配的账号进入工作台</p>
</div>
<label>用户名<input id="loginUsername" autocomplete="username" required></label>
<label>密码<input id="loginPassword" type="password" autocomplete="current-password" required></label>
<p id="loginError" class="form-error" role="alert"></p>
<button class="primary-button" type="submit">登录</button>
</form>
<form id="passwordForm" class="auth-card" hidden>
<div>
<h1>设置新密码</h1>
<p>首次登录需要完成密码更新</p>
</div>
<label>当前密码<input id="currentPassword" type="password" autocomplete="current-password" required></label>
<label>新密码<input id="newPassword" type="password" autocomplete="new-password" minlength="12" required></label>
<label>确认新密码<input id="confirmPassword" type="password" autocomplete="new-password" minlength="12" required></label>
<p class="password-hint">至少 12 位,并包含大小写字母、数字或符号中的三类。</p>
<p id="passwordError" class="form-error" role="alert"></p>
<button class="primary-button" type="submit">保存并重新登录</button>
</form>
</section>
<div id="appShell" class="app-shell" hidden>
<aside class="sidebar">
<div class="brand">
<strong>Real2Sim</strong>
@ -23,6 +56,11 @@
<button class="tab" data-tab="train">模型训练</button>
<button class="tab" data-tab="infer">在线推理</button>
</nav>
<div class="sidebar-actions">
<button id="openUsers" hidden>用户管理</button>
<button id="logoutButton">退出登录</button>
<span id="currentUserLabel"></span>
</div>
</aside>
<main class="main-surface">
@ -106,9 +144,9 @@
<strong>上传文件</strong>
<span>zip、视频或图片包</span>
</button>
<button class="source-card" data-open-import="local">
<strong>本地目录</strong>
<span>从配置白名单导入已有数据</span>
<button class="source-card" data-open-import="bos">
<strong>BOS 存储</strong>
<span>从授权的 Real2Sim 前缀导入</span>
</button>
</div>
@ -228,7 +266,6 @@
<label class="check"><input id="replaceSceneWithPoseTrain" type="checkbox" checked> 用训练结果替换主场景</label>
<label class="check"><input id="exportTextures" type="checkbox" checked> 生成纹理资产</label>
<label class="check"><input id="keepIntermediate" type="checkbox" checked> 保留中间产物</label>
<label class="check muted-check"><input id="dryRunScene" type="checkbox"> 仅生成样例</label>
</div>
</details>
@ -362,10 +399,10 @@
</div>
<div class="form-grid">
<label class="span-2">选择场景<select id="collectScene"></select></label>
<label>专家策略<select id="policyPreset"><option value="pick_and_place_v1">抓取放置策略</option><option value="navigation_v1" disabled>导航采集策略(待接入)</option><option value="manipulation_v2" disabled>精细操作策略(待接入)</option></select></label>
<label>专家控制器<select id="policyPreset"><option value="">正在读取可用控制器</option></select></label>
<label>执行并发<input id="collectWorkers" type="number" min="1" max="1" value="1" disabled title="当前专家采集执行器仅支持单 worker"></label>
<label>Episode 数<input id="episodes" type="number" value="4"></label>
<label>每个 Episode 步数<input id="steps" type="number" value="24"></label>
<label>成功 Episode 数<input id="episodes" type="number" min="1" value="50"></label>
<label>每个 Episode 步数<input id="steps" type="number" min="80" value="240"></label>
</div>
</section>
@ -417,7 +454,7 @@
</div>
<div class="form-grid">
<label class="span-2">训练数据集<select id="trainDataset"></select></label>
<label>训练配方<select id="trainRecipe"><option value="bc_v2">行为克隆 v2</option><option value="bc_fast">快速验证</option><option value="diffusion_policy" disabled>扩散策略(待接入)</option></select></label>
<label>训练配方<select id="trainRecipe"><option value="">正在读取可用配方</option></select></label>
<label>Epoch 数<input id="epochs" type="number" value="100"></label>
</div>
</section>
@ -488,7 +525,8 @@
<div class="form-grid">
<label>选择模型<select id="inferModel"></select></label>
<label>选择场景<select id="inferScene"></select></label>
<label>步数上限<input id="inferSteps" type="number" min="1" value="80"></label>
<label>推理 Episode 数<input id="inferEpisodes" type="number" min="1" value="10"></label>
<label>步数上限<input id="inferSteps" type="number" min="80" value="240"></label>
<label>执行并发<input id="inferWorkers" type="number" min="1" max="1" value="1" disabled title="当前推理执行器仅支持单 worker"></label>
</div>
</section>
@ -580,7 +618,23 @@
</form>
</dialog>
<dialog id="usersDialog">
<div class="dialog-card">
<div class="panel-head">
<div><h2>用户管理</h2><p>创建、禁用和重置团队账号</p></div>
<button type="button" data-close-dialog="usersDialog" class="ghost-button">关闭</button>
</div>
<form id="createUserForm" class="form-grid user-form">
<label>用户名<input id="newUsername" required></label>
<label>角色<select id="newUserRole"><option value="engineer">工程师</option><option value="viewer">只读用户</option><option value="admin">管理员</option></select></label>
<label class="span-2">初始密码<input id="newUserPassword" type="password" minlength="12" required></label>
<button class="primary-button span-2" type="submit">创建用户</button>
</form>
<div id="userList" class="user-list"></div>
</div>
</dialog>
<div id="toast" class="toast" role="status"></div>
<script src="/static/app.js?v=20260710-motrix-runtime-v7"></script>
<script src="/static/app.js?v=20260710-poc-product-v1"></script>
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,182 @@
from __future__ import annotations
import hashlib
import os
import re
from pathlib import Path, PurePosixPath
from typing import Any, Iterable
from .config import PlatformConfig
ALLOWED_AREAS = {"uploads", "datasets", "scenes", "models", "runs", "backups"}
_SAFE_COMPONENT = re.compile(r"[^a-zA-Z0-9._-]+")
class StorageUnavailable(RuntimeError):
pass
class StorageBoundaryError(ValueError):
pass
class BosStorage:
def __init__(self, config: PlatformConfig) -> None:
self.config = config
self.bucket = config.bos_bucket.strip()
self.prefix = normalize_prefix(config.bos_prefix)
self._client = None
@property
def configured(self) -> bool:
return bool(self.bucket and self.config.bos_access_key_id and self.config.bos_secret_access_key)
def health(self) -> dict[str, Any]:
result = {"configured": self.configured, "bucket": self.bucket, "prefix": self.prefix, "endpoint": self.config.bos_endpoint}
if not self.configured:
result["ready"] = False
result["reason"] = "BOS credentials or bucket are not configured"
return result
try:
self.list_objects(area="uploads", relative_prefix="", max_keys=1)
result["ready"] = True
except Exception as exc:
result["ready"] = False
result["reason"] = str(exc)
return result
def object_key(self, area: str, relative_path: str) -> str:
if area not in ALLOWED_AREAS:
raise StorageBoundaryError(f"unsupported BOS area: {area}")
relative = normalize_relative_path(relative_path)
return f"{self.prefix}/{area}/{relative}" if relative else f"{self.prefix}/{area}/"
def uri_for_key(self, key: str) -> str:
checked = self.assert_owned_key(key)
return f"bos://{self.bucket}/{checked}"
def assert_owned_key(self, key: str) -> str:
normalized = str(PurePosixPath(str(key).strip().lstrip("/")))
owned_root = f"{self.prefix}/"
if not normalized.startswith(owned_root) or normalized == self.prefix:
raise StorageBoundaryError("object key is outside the Real2Sim BOS prefix")
area = normalized[len(owned_root):].split("/", 1)[0]
if area not in ALLOWED_AREAS:
raise StorageBoundaryError("object key is outside an approved Real2Sim area")
if ".." in PurePosixPath(normalized).parts:
raise StorageBoundaryError("parent traversal is not allowed")
return normalized
def upload_file(self, local_path: Path, *, area: str, relative_path: str, metadata: dict[str, str] | None = None) -> dict[str, Any]:
if not local_path.is_file():
raise FileNotFoundError(local_path)
key = self.object_key(area, relative_path)
client = self._get_client()
kwargs: dict[str, Any] = {}
if metadata:
kwargs["user_metadata"] = {str(k): str(v) for k, v in metadata.items()}
client.put_object_from_file(self.bucket, key, str(local_path), **kwargs)
return {
"key": key,
"storage_uri": self.uri_for_key(key),
"size_bytes": local_path.stat().st_size,
"checksum": sha256_file(local_path),
}
def download_file(self, storage_uri: str, target_path: Path) -> Path:
key = self.key_from_uri(storage_uri)
target_path.parent.mkdir(parents=True, exist_ok=True)
self._get_client().get_object_to_file(self.bucket, key, str(target_path))
return target_path
def list_objects(self, *, area: str, relative_prefix: str = "", max_keys: int = 1000) -> list[dict[str, Any]]:
prefix = self.object_key(area, relative_prefix)
response = self._get_client().list_objects(self.bucket, prefix=prefix, max_keys=max(1, min(1000, int(max_keys))))
contents: Iterable[Any] = getattr(response, "contents", None) or []
return [
{
"key": self.assert_owned_key(str(getattr(item, "key", ""))),
"storage_uri": self.uri_for_key(str(getattr(item, "key", ""))),
"size_bytes": int(getattr(item, "size", 0) or 0),
"etag": str(getattr(item, "e_tag", getattr(item, "etag", "")) or "").strip('"'),
"last_modified": str(getattr(item, "last_modified", "") or ""),
}
for item in contents
if getattr(item, "key", None)
]
def delete_object(self, storage_uri: str) -> None:
key = self.key_from_uri(storage_uri)
self._get_client().delete_object(self.bucket, key)
def key_from_uri(self, storage_uri: str) -> str:
prefix = f"bos://{self.bucket}/"
if not storage_uri.startswith(prefix):
raise StorageBoundaryError("BOS URI bucket does not match configured bucket")
return self.assert_owned_key(storage_uri[len(prefix):])
def _get_client(self):
if not self.configured:
raise StorageUnavailable("BOS storage is not configured")
if self._client is not None:
return self._client
try:
from baidubce.auth.bce_credentials import BceCredentials
from baidubce.bce_client_configuration import BceClientConfiguration
from baidubce.services.bos.bos_client import BosClient
except ImportError as exc:
raise StorageUnavailable("bce-python-sdk is not installed") from exc
credentials = BceCredentials(self.config.bos_access_key_id, self.config.bos_secret_access_key)
configuration = BceClientConfiguration(credentials=credentials, endpoint=self.config.bos_endpoint)
self._client = BosClient(configuration)
return self._client
def normalize_prefix(value: str) -> str:
parts = PurePosixPath(str(value).strip().strip("/"))
if not str(parts) or str(parts) == "." or ".." in parts.parts:
raise StorageBoundaryError("BOS prefix must be a non-empty relative path")
return str(parts)
def normalize_relative_path(value: str) -> str:
raw = str(value or "").replace("\\", "/").strip().lstrip("/")
if not raw:
return ""
path = PurePosixPath(raw)
if path.is_absolute() or ".." in path.parts:
raise StorageBoundaryError("BOS path traversal is not allowed")
return str(path)
def safe_object_name(filename: str) -> str:
name = Path(filename).name.strip()
cleaned = _SAFE_COMPONENT.sub("-", name).strip(".-")
if not cleaned:
raise ValueError("filename is empty after sanitization")
return cleaned[:180]
def sha256_file(path: Path, chunk_size: int = 8 * 1024 * 1024) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
while chunk := handle.read(chunk_size):
digest.update(chunk)
return digest.hexdigest()
def atomic_write_part(path: Path, chunks: Iterable[bytes]) -> tuple[int, str]:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
digest = hashlib.sha256()
size = 0
with temporary.open("wb") as handle:
for chunk in chunks:
handle.write(chunk)
digest.update(chunk)
size += len(chunk)
handle.flush()
os.fsync(handle.fileno())
temporary.replace(path)
return size, digest.hexdigest()

View File

@ -28,13 +28,16 @@ class PlatformStore:
manifest_path: Path | None = None,
metrics: dict[str, Any] | None = None,
tags: list[str] | None = None,
storage_uri: str = "",
checksum: str = "",
cache_state: str = "local",
) -> dict[str, Any]:
item_id = item_id or uuid.uuid4().hex
now = _now()
with self._connect() as db:
db.execute(
"insert into datasets(id,name,path,scan_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)",
(item_id, name, str(path), _json(scan), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now),
"insert into datasets(id,name,path,scan_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(item_id, name, str(path), _json(scan), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now),
)
return self.get_dataset(item_id)
@ -65,13 +68,16 @@ class PlatformStore:
manifest_path: Path | None = None,
metrics: dict[str, Any] | None = None,
tags: list[str] | None = None,
storage_uri: str = "",
checksum: str = "",
cache_state: str = "local",
) -> dict[str, Any]:
item_id = item_id or uuid.uuid4().hex
now = _now()
with self._connect() as db:
db.execute(
"insert into scenes(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)",
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now),
"insert into scenes(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now),
)
return self.get_scene(item_id)
@ -125,13 +131,16 @@ class PlatformStore:
manifest_path: Path | None = None,
metrics: dict[str, Any] | None = None,
tags: list[str] | None = None,
storage_uri: str = "",
checksum: str = "",
cache_state: str = "local",
) -> dict[str, Any]:
item_id = item_id or uuid.uuid4().hex
now = _now()
with self._connect() as db:
db.execute(
"insert into models(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)",
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now),
"insert into models(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now),
)
return self.get_model(item_id)
@ -148,6 +157,18 @@ class PlatformStore:
def delete_model(self, item_id: str) -> None:
self._delete("models", item_id)
def update_entity_storage(self, table: str, item_id: str, *, storage_uri: str, checksum: str = "", cache_state: str = "local") -> dict[str, Any]:
if table not in {"datasets", "scenes", "models"}:
raise ValueError("unsupported entity table")
mapper = {"datasets": self._dataset_row, "scenes": self._scene_row, "models": self._model_row}[table]
self._get(table, item_id, mapper)
with self._connect() as db:
db.execute(
f"update {table} set storage_uri=?,checksum=?,cache_state=?,updated_at=? where id=?",
(storage_uri, checksum, cache_state, _now(), item_id),
)
return self._get(table, item_id, mapper)
def create_task(self, *, task_type: str, params: dict[str, Any], run_dir: Path) -> dict[str, Any]:
item_id = uuid.uuid4().hex
now = _now()
@ -164,7 +185,33 @@ class PlatformStore:
def get_task(self, item_id: str) -> dict[str, Any]:
return self._get("tasks", item_id, self._task_row)
def claim_next_queued_task(self, worker_id: str) -> dict[str, Any] | None:
db = self._connect()
try:
db.execute("begin immediate")
row = db.execute("select * from tasks where status='queued' order by created_at limit 1").fetchone()
if row is None:
db.commit()
return None
now = _now()
cursor = db.execute(
"update tasks set status='running',worker_id=?,claimed_at=?,attempt=attempt+1,progress=max(progress,1),stage='启动中',message='GPU worker 已领取任务',updated_at=? where id=? and status='queued'",
(worker_id, now, now, row["id"]),
)
if cursor.rowcount != 1:
db.rollback()
return None
claimed = db.execute("select * from tasks where id=?", (row["id"],)).fetchone()
db.commit()
return self._task_row(claimed)
except Exception:
db.rollback()
raise
finally:
db.close()
def next_queued_task(self) -> dict[str, Any] | None:
"""Compatibility helper for older callers; new workers must claim atomically."""
with self._connect() as db:
row = db.execute("select * from tasks where status='queued' order by created_at limit 1").fetchone()
return self._task_row(row) if row else None
@ -258,6 +305,153 @@ class PlatformStore:
)
return self.get_artifact(item_id)
def create_user(self, *, username: str, password_hash: str, role: str, must_change_password: bool = True) -> dict[str, Any]:
item_id = uuid.uuid4().hex
now = _now()
with self._connect() as db:
db.execute(
"insert into users(id,username,password_hash,role,disabled,must_change_password,created_at,updated_at) values(?,?,?,?,0,?,?,?)",
(item_id, username, password_hash, role, int(must_change_password), now, now),
)
return self.get_user(item_id)
def count_users(self) -> int:
with self._connect() as db:
return int(db.execute("select count(*) from users").fetchone()[0])
def list_users(self) -> list[dict[str, Any]]:
with self._connect() as db:
rows = db.execute("select * from users order by created_at").fetchall()
return [self._user_row(row) for row in rows]
def get_user(self, user_id: str) -> dict[str, Any]:
return self._get("users", user_id, self._user_row)
def get_user_by_username(self, username: str) -> dict[str, Any] | None:
with self._connect() as db:
row = db.execute("select * from users where lower(username)=lower(?)", (username,)).fetchone()
return self._user_row(row, include_password=True) if row else None
def update_user(
self,
user_id: str,
*,
role: str | None = None,
disabled: bool | None = None,
password_hash: str | None = None,
must_change_password: bool | None = None,
last_login: float | None = None,
) -> dict[str, Any]:
self.get_user(user_id)
fields: list[str] = []
values: list[Any] = []
for name, value in (("role", role), ("password_hash", password_hash), ("last_login", last_login)):
if value is not None:
fields.append(f"{name}=?")
values.append(value)
for name, value in (("disabled", disabled), ("must_change_password", must_change_password)):
if value is not None:
fields.append(f"{name}=?")
values.append(int(value))
if fields:
fields.append("updated_at=?")
values.extend([_now(), user_id])
with self._connect() as db:
db.execute(f"update users set {','.join(fields)} where id=?", values)
return self.get_user(user_id)
def delete_user(self, user_id: str) -> None:
self._delete("users", user_id)
def create_session(self, *, token_hash: str, user_id: str, expires_at: float) -> None:
now = _now()
with self._connect() as db:
db.execute(
"insert into sessions(token_hash,user_id,expires_at,created_at,last_seen) values(?,?,?,?,?)",
(token_hash, user_id, expires_at, now, now),
)
def get_session_user(self, token_hash: str) -> dict[str, Any] | None:
now = _now()
with self._connect() as db:
row = db.execute(
"select users.* from sessions join users on users.id=sessions.user_id where sessions.token_hash=? and sessions.expires_at>? and users.disabled=0",
(token_hash, now),
).fetchone()
if row:
db.execute("update sessions set last_seen=? where token_hash=?", (now, token_hash))
db.execute("delete from sessions where expires_at<=?", (now,))
return self._user_row(row) if row else None
def delete_session(self, token_hash: str) -> None:
with self._connect() as db:
db.execute("delete from sessions where token_hash=?", (token_hash,))
def delete_user_sessions(self, user_id: str) -> None:
with self._connect() as db:
db.execute("delete from sessions where user_id=?", (user_id,))
def create_upload(
self,
*,
owner_user_id: str | None,
filename: str,
content_type: str,
total_size: int,
chunk_size: int,
total_parts: int,
checksum_expected: str,
local_dir: Path,
item_id: str | None = None,
) -> dict[str, Any]:
item_id = item_id or uuid.uuid4().hex
now = _now()
with self._connect() as db:
db.execute(
"insert into uploads(id,owner_user_id,filename,content_type,total_size,chunk_size,total_parts,received_parts_json,checksum_expected,checksum_actual,status,local_dir,storage_uri,error,created_at,updated_at) values(?,?,?,?,?,?,?,'[]',?,?,'uploading',?,'','',?,?)",
(item_id, owner_user_id, filename, content_type, total_size, chunk_size, total_parts, checksum_expected, "", str(local_dir), now, now),
)
return self.get_upload(item_id)
def get_upload(self, upload_id: str) -> dict[str, Any]:
return self._get("uploads", upload_id, self._upload_row)
def record_upload_part(self, upload_id: str, part_number: int) -> dict[str, Any]:
upload = self.get_upload(upload_id)
parts = sorted(set([*upload["received_parts"], int(part_number)]))
with self._connect() as db:
db.execute("update uploads set received_parts_json=?,updated_at=? where id=?", (_json(parts), _now(), upload_id))
return self.get_upload(upload_id)
def complete_upload(self, upload_id: str, *, checksum_actual: str, storage_uri: str) -> dict[str, Any]:
with self._connect() as db:
db.execute(
"update uploads set status='completed',checksum_actual=?,storage_uri=?,error='',updated_at=? where id=?",
(checksum_actual, storage_uri, _now(), upload_id),
)
return self.get_upload(upload_id)
def fail_upload(self, upload_id: str, error: str) -> dict[str, Any]:
with self._connect() as db:
db.execute("update uploads set status='failed',error=?,updated_at=? where id=?", (error, _now(), upload_id))
return self.get_upload(upload_id)
def delete_upload(self, upload_id: str) -> None:
self._delete("uploads", upload_id)
def update_worker_heartbeat(self, worker_id: str, payload: dict[str, Any]) -> None:
now = _now()
with self._connect() as db:
db.execute(
"insert into worker_heartbeats(worker_id,payload_json,updated_at) values(?,?,?) on conflict(worker_id) do update set payload_json=excluded.payload_json,updated_at=excluded.updated_at",
(worker_id, _json(payload), now),
)
def list_worker_heartbeats(self) -> list[dict[str, Any]]:
with self._connect() as db:
rows = db.execute("select * from worker_heartbeats order by updated_at desc").fetchall()
return [{"worker_id": row["worker_id"], "payload": json.loads(row["payload_json"] or "{}"), "updated_at": row["updated_at"]} for row in rows]
def list_task_artifacts(self, task_id: str) -> list[dict[str, Any]]:
with self._connect() as db:
rows = db.execute("select * from artifacts where task_id=? order by created_at", (task_id,)).fetchall()
@ -268,6 +462,8 @@ class PlatformStore:
def _init(self) -> None:
with self._connect() as db:
db.execute("pragma journal_mode=WAL")
db.execute("pragma synchronous=NORMAL")
db.execute("create table if not exists datasets(id text primary key,name text,path text,scan_json text,created_at real,updated_at real)")
db.execute("create table if not exists scenes(id text primary key,name text,path text,metadata_json text,created_at real,updated_at real)")
db.execute("create table if not exists models(id text primary key,name text,path text,metadata_json text,created_at real,updated_at real)")
@ -277,6 +473,16 @@ class PlatformStore:
"run_dir text,error text,cancel_requested integer not null default 0,created_at real,updated_at real)"
)
db.execute("create table if not exists artifacts(id text primary key,task_id text,name text,path text,kind text,created_at real)")
db.execute(
"create table if not exists users(id text primary key,username text not null unique,password_hash text not null,role text not null,disabled integer not null default 0,must_change_password integer not null default 1,last_login real,created_at real,updated_at real)"
)
db.execute(
"create table if not exists sessions(token_hash text primary key,user_id text not null,expires_at real not null,created_at real,last_seen real,foreign key(user_id) references users(id) on delete cascade)"
)
db.execute(
"create table if not exists uploads(id text primary key,owner_user_id text,filename text not null,content_type text,total_size integer not null,chunk_size integer not null,total_parts integer not null,received_parts_json text not null default '[]',checksum_expected text,checksum_actual text,status text not null,local_dir text not null,storage_uri text,error text,created_at real,updated_at real)"
)
db.execute("create table if not exists worker_heartbeats(worker_id text primary key,payload_json text not null,updated_at real not null)")
for table in ["datasets", "scenes", "models"]:
self._ensure_column(db, table, "source_type", "text not null default 'local'")
self._ensure_column(db, table, "source_uri", "text not null default ''")
@ -285,14 +491,24 @@ class PlatformStore:
self._ensure_column(db, table, "manifest_path", "text not null default ''")
self._ensure_column(db, table, "metrics_json", "text not null default '{}'")
self._ensure_column(db, table, "tags_json", "text not null default '[]'")
self._ensure_column(db, table, "storage_uri", "text not null default ''")
self._ensure_column(db, table, "checksum", "text not null default ''")
self._ensure_column(db, table, "cache_state", "text not null default 'local'")
self._ensure_column(db, "tasks", "pid", "integer")
self._ensure_column(db, "tasks", "progress", "integer not null default 0")
self._ensure_column(db, "tasks", "stage", "text not null default ''")
self._ensure_column(db, "tasks", "message", "text not null default ''")
self._ensure_column(db, "tasks", "worker_id", "text not null default ''")
self._ensure_column(db, "tasks", "claimed_at", "real")
self._ensure_column(db, "tasks", "attempt", "integer not null default 0")
db.execute("create index if not exists idx_tasks_status_created on tasks(status,created_at)")
db.execute("create index if not exists idx_sessions_user on sessions(user_id)")
def _connect(self) -> sqlite3.Connection:
db = sqlite3.connect(self.db_path)
db = sqlite3.connect(self.db_path, timeout=30)
db.row_factory = sqlite3.Row
db.execute("pragma busy_timeout=5000")
db.execute("pragma foreign_keys=on")
return db
def _ensure_column(self, db: sqlite3.Connection, table: str, column: str, spec: str) -> None:
@ -359,6 +575,9 @@ class PlatformStore:
"manifest_path": row["manifest_path"],
"metrics": json.loads(row["metrics_json"] or "{}"),
"tags": json.loads(row["tags_json"] or "[]"),
"storage_uri": row["storage_uri"],
"checksum": row["checksum"],
"cache_state": row["cache_state"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
@ -377,6 +596,9 @@ class PlatformStore:
"message": row["message"] or "",
"cancel_requested": bool(row["cancel_requested"]),
"pid": row["pid"],
"worker_id": row["worker_id"],
"claimed_at": row["claimed_at"],
"attempt": int(row["attempt"] or 0),
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
@ -384,6 +606,41 @@ class PlatformStore:
def _artifact_row(self, row: sqlite3.Row) -> dict[str, Any]:
return {"id": row["id"], "task_id": row["task_id"], "name": row["name"], "path": row["path"], "kind": row["kind"], "created_at": row["created_at"]}
def _user_row(self, row: sqlite3.Row, *, include_password: bool = False) -> dict[str, Any]:
item = {
"id": row["id"],
"username": row["username"],
"role": row["role"],
"disabled": bool(row["disabled"]),
"must_change_password": bool(row["must_change_password"]),
"last_login": row["last_login"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
if include_password:
item["password_hash"] = row["password_hash"]
return item
def _upload_row(self, row: sqlite3.Row) -> dict[str, Any]:
return {
"id": row["id"],
"owner_user_id": row["owner_user_id"],
"filename": row["filename"],
"content_type": row["content_type"] or "application/octet-stream",
"total_size": int(row["total_size"]),
"chunk_size": int(row["chunk_size"]),
"total_parts": int(row["total_parts"]),
"received_parts": json.loads(row["received_parts_json"] or "[]"),
"checksum_expected": row["checksum_expected"] or "",
"checksum_actual": row["checksum_actual"] or "",
"status": row["status"],
"local_dir": row["local_dir"],
"storage_uri": row["storage_uri"] or "",
"error": row["error"] or "",
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
def _json(value: Any) -> str:
return json.dumps(value, ensure_ascii=False)

View File

@ -7,7 +7,7 @@ from typing import Any
from pathlib import Path
from .adapters import TaskCanceled, execute_task, index_artifacts_for_task
from .adapters import TaskCanceled, execute_task, index_artifacts_for_task, run_doctor, sync_task_outputs
from .config import PlatformConfig
from .store import PlatformStore
@ -18,40 +18,65 @@ class TaskWorker:
self.config = config
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._last_heartbeat = 0.0
def start(self) -> None:
if self._thread and self._thread.is_alive():
return
self._stop.clear()
self.store.recover_interrupted_tasks()
self._heartbeat(force=True)
self._thread = threading.Thread(target=self._loop, name="real2sim-platform-worker", daemon=True)
self._thread.start()
def stop(self) -> None:
self._stop.set()
if self._thread and self._thread.is_alive():
self._thread.join(timeout=5)
def _loop(self) -> None:
while not self._stop.is_set():
task = self.store.next_queued_task()
self._heartbeat()
task = self.store.claim_next_queued_task(self.config.worker_id)
if task is None:
time.sleep(0.5)
continue
self._run(task)
def _heartbeat(self, *, force: bool = False) -> None:
now = time.time()
if not force and now - self._last_heartbeat < 10:
return
report = run_doctor(self.config)
self.store.update_worker_heartbeat(
self.config.worker_id,
{
"ready": bool(report.get("ok")),
"checks": report.get("checks", {}),
"warnings": report.get("warnings", []),
"pid": __import__("os").getpid(),
},
)
self._last_heartbeat = now
def _run(self, task: dict[str, Any]) -> None:
if self.store.get_task(task["id"])["cancel_requested"]:
self.store.update_task(task["id"], status="canceled", error="task canceled before start", progress=0, stage="已取消", message="任务启动前已取消")
return
self.store.update_task(task["id"], status="running", progress=5, stage="启动中", message="任务已进入执行队列")
self.store.update_task(task["id"], status="running", progress=5, stage="启动中", message=f"{self.config.worker_id} 正在初始化任务")
try:
result = execute_task(self.config, self.store, task)
result = sync_task_outputs(self.config, self.store, task, result)
except TaskCanceled as exc:
self._write_trace(task, exc)
index_artifacts_for_task(self.store, task["id"], Path(task["run_dir"]))
self._persist_failed_run(task)
self.store.update_task(task["id"], status="canceled", error=str(exc), progress=0, stage="已取消", message=str(exc))
return
except Exception as exc:
self._write_trace(task, exc)
index_artifacts_for_task(self.store, task["id"], Path(task["run_dir"]))
self._persist_failed_run(task)
self.store.update_task(task["id"], status="failed", error=str(exc), stage="执行失败", message=str(exc))
return
status = "canceled" if self.store.get_task(task["id"])["cancel_requested"] else "succeeded"
@ -65,3 +90,11 @@ class TaskWorker:
with log_path.open("a", encoding="utf-8") as handle:
handle.write("\nERROR: " + str(exc) + "\n")
handle.write(traceback.format_exc() + "\n")
def _persist_failed_run(self, task: dict[str, Any]) -> None:
try:
sync_task_outputs(self.config, self.store, task, {})
except Exception as exc:
log_path = Path(task["run_dir"]) / "logs" / "task.log"
with log_path.open("a", encoding="utf-8") as handle:
handle.write(f"\nfailed to persist terminal task artifacts: {exc}\n")

View File

@ -0,0 +1,37 @@
from __future__ import annotations
import argparse
import signal
import time
from pathlib import Path
from .config import ensure_workspace, load_config
from .store import PlatformStore
from .worker import TaskWorker
def main() -> int:
parser = argparse.ArgumentParser(description="Real2Sim serial GPU worker")
parser.add_argument("--config", type=Path)
args = parser.parse_args()
config = load_config(args.config)
ensure_workspace(config)
store = PlatformStore(config.db_path)
worker = TaskWorker(store, config)
stopping = False
def stop(*_args) -> None:
nonlocal stopping
stopping = True
worker.stop()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
worker.start()
while not stopping:
time.sleep(1)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -215,11 +215,14 @@ def test_scene_asset_append_updates_manifest_and_artifact(tmp_path: Path) -> Non
assert response.status_code == 200, response.text
payload = response.json()
assert payload["asset"]["name"] == "fixture metadata"
manifest = client.get(f"/api/scenes/{scene['id']}/manifest").json()
assert payload["scene"]["id"] != scene["id"]
assert payload["parent_scene_id"] == scene["id"]
manifest = client.get(f"/api/scenes/{payload['scene']['id']}/manifest").json()
added = [item for item in manifest["assets"] if item["name"] == "fixture metadata"]
assert added
assert added[0]["role"] == "simulator_import"
assert added[0]["transform"]["translation"] == [1, 2, 3]
assert manifest["revision"]["parent_scene_id"] == scene["id"]
assert client.get(added[0]["url"]).status_code == 200

View File

@ -0,0 +1,185 @@
from __future__ import annotations
import concurrent.futures
from pathlib import Path
import h5py
import pytest
from fastapi.testclient import TestClient
from real2sim_platform.config import PlatformConfig
from real2sim_platform.rm65_controller import collect
from real2sim_platform.server import create_app
from real2sim_platform.storage import BosStorage, StorageBoundaryError, normalize_relative_path
from real2sim_platform.store import PlatformStore
ROOT = Path(__file__).resolve().parents[1]
def test_auth_first_login_rbac_and_user_crud(tmp_path: Path) -> None:
config = PlatformConfig(
app_root=ROOT,
workspace_dir=tmp_path / "workspace",
worker_mode="api",
auth_enabled=True,
bootstrap_admin_password="Admin!Password123",
)
with TestClient(create_app(config)) as client:
assert client.get("/api/datasets").status_code == 401
login = client.post("/api/auth/login", json={"username": "admin", "password": "Admin!Password123"})
assert login.status_code == 200
assert login.json()["user"]["must_change_password"] is True
assert client.get("/api/datasets").status_code == 428
changed = client.post(
"/api/auth/change-password",
json={"current_password": "Admin!Password123", "new_password": "Changed!Password456"},
)
assert changed.status_code == 200
assert client.get("/api/datasets").status_code == 401
assert client.post("/api/auth/login", json={"username": "admin", "password": "Changed!Password456"}).status_code == 200
viewer = client.post(
"/api/users",
json={"username": "viewer.user", "password": "Viewer!Password123", "role": "viewer"},
)
assert viewer.status_code == 200
assert viewer.json()["role"] == "viewer"
assert client.patch(f"/api/users/{viewer.json()['id']}", json={"disabled": True}).json()["disabled"] is True
viewer_config = PlatformConfig(
app_root=ROOT,
workspace_dir=config.workspace_dir,
worker_mode="api",
auth_enabled=True,
bootstrap_admin_password="unused-Password!123",
)
store = PlatformStore(viewer_config.db_path)
viewer_id = next(item["id"] for item in store.list_users() if item["username"] == "viewer.user")
store.update_user(viewer_id, disabled=False)
with TestClient(create_app(viewer_config)) as viewer_client:
assert viewer_client.post("/api/auth/login", json={"username": "viewer.user", "password": "Viewer!Password123"}).status_code == 200
assert viewer_client.get("/api/auth/me").status_code == 200
assert viewer_client.post("/api/auth/change-password", json={"current_password": "Viewer!Password123", "new_password": "Viewer!Changed456"}).status_code == 200
assert viewer_client.post("/api/auth/login", json={"username": "viewer.user", "password": "Viewer!Changed456"}).status_code == 200
assert viewer_client.get("/api/datasets").status_code == 200
assert viewer_client.post("/api/tasks", json={"type": "renderer_benchmark", "params": {}}).status_code == 403
def test_bos_prefix_boundary_is_enforced(tmp_path: Path) -> None:
config = PlatformConfig(
workspace_dir=tmp_path,
bos_bucket="testxyz",
bos_prefix="real2sim-poc",
bos_access_key_id="configured-but-not-used",
bos_secret_access_key="configured-but-not-used",
)
storage = BosStorage(config)
assert storage.object_key("datasets", "team/a.json") == "real2sim-poc/datasets/team/a.json"
assert storage.key_from_uri("bos://testxyz/real2sim-poc/models/m1/card.json") == "real2sim-poc/models/m1/card.json"
with pytest.raises(StorageBoundaryError):
storage.object_key("datasets", "../../droboticflow/secret")
with pytest.raises(StorageBoundaryError):
storage.assert_owned_key("droboticflow/workspaces/data")
with pytest.raises(StorageBoundaryError):
storage.key_from_uri("bos://another-bucket/real2sim-poc/datasets/a")
with pytest.raises(StorageBoundaryError):
normalize_relative_path("../outside")
def test_atomic_task_claim_has_no_duplicates(tmp_path: Path) -> None:
store = PlatformStore(tmp_path / "state.db")
for index in range(24):
store.create_task(task_type="renderer_benchmark", params={"index": index}, run_dir=tmp_path / f"run-{index}")
def claim(worker: int):
task = store.claim_next_queued_task(f"worker-{worker}")
return task["id"] if task else None
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
claimed = [item for item in executor.map(claim, range(24)) if item]
assert len(claimed) == 24
assert len(set(claimed)) == 24
assert all(task["status"] == "running" and task["attempt"] == 1 for task in store.list_tasks())
def test_api_capabilities_follow_live_worker_heartbeat(tmp_path: Path) -> None:
config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api")
store = PlatformStore(config.db_path)
store.update_worker_heartbeat(
"worker-a800",
{
"ready": True,
"checks": {
"ffmpeg": True,
"colmap": True,
"anysplat_root": True,
"anysplat_python": True,
"anysplat_pinned": True,
"mujoco_python": True,
"motrixsim_python": True,
"discoverse_act": True,
"torch_cuda": True,
"act_import": True,
"rm65_controller": True,
},
},
)
with TestClient(create_app(config)) as client:
capabilities = client.get("/api/capabilities").json()
assert capabilities["worker"]["connected"] is True
assert capabilities["runtimes"]["anysplat"]["ready"] is True
assert capabilities["runtimes"]["discoverse_act"]["ready"] is True
assert capabilities["runtimes"]["rm65_controller"]["ready"] is True
def test_login_rate_limit_blocks_repeated_failures(tmp_path: Path) -> None:
config = PlatformConfig(
app_root=ROOT,
workspace_dir=tmp_path / "workspace",
worker_mode="api",
auth_enabled=True,
bootstrap_admin_password="Admin!Password123",
)
with TestClient(create_app(config)) as client:
for _ in range(8):
assert client.post("/api/auth/login", json={"username": "admin", "password": "wrong-password"}).status_code == 401
assert client.post("/api/auth/login", json={"username": "admin", "password": "wrong-password"}).status_code == 429
def test_production_task_gates_reject_fake_routes(tmp_path: Path) -> None:
config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api", production_mode=True)
client = TestClient(create_app(config))
dry_run = client.post(
"/api/tasks",
json={"type": "scene_reconstruction", "params": {"input_path": str(tmp_path), "dry_run": True}},
)
assert dry_run.status_code == 400
legacy = client.post(
"/api/tasks",
json={"type": "training", "params": {"dataset_path": str(tmp_path), "recipe": "bc_v2"}},
)
assert legacy.status_code == 400
fake_expert = client.post(
"/api/tasks",
json={"type": "expert_collection", "params": {"scene_path": str(tmp_path), "policy_preset": "pick_and_place_v1"}},
)
assert fake_expert.status_code == 400
@pytest.mark.integration
def test_real_rm65_physical_collection_writes_act_hdf5(tmp_path: Path) -> None:
scene = ROOT / "workspace/motrixsim_assets/parlour_rm65_pick_place/simulator_assets/rm65_table_pick_place.xml"
if not scene.is_file():
pytest.skip("real imported RM65 scene asset is not present")
manifest = collect(scene, tmp_path / "collection", required_successes=1, max_attempts=2, max_steps=240, seed=7, width=160, height=120)
assert manifest["successful_episodes"] == 1
assert manifest["episodes"][0]["lift_height"] >= 0.10
assert manifest["episodes"][0]["place_error"] <= 0.05
episode = tmp_path / "collection/episode_0.hdf5"
with h5py.File(episode, "r") as root:
assert root["/observations/qpos"].shape[1] == 7
assert root["/action"].shape[1] == 7
assert root["/observations/images/overview"].shape[0] > 0
assert root["/observations/contact"].shape[1] == 2
assert bool(root.attrs["success"]) is True
assert (tmp_path / "collection/episode_0.mp4").stat().st_size > 0

21
vendor/gs_playground/LICENSE vendored Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Yufei Jia (贾宇飞)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1
vendor/gs_playground/UPSTREAM_COMMIT vendored Normal file
View File

@ -0,0 +1 @@
e10805930b2b652f5608c41f671c3460b47c0c72

View File

@ -0,0 +1,9 @@
"""Real2Sim pipeline tools for GS-Playground."""
from .errors import PipelineError, MissingStageCommand, StageArtifactError
__all__ = [
"PipelineError",
"MissingStageCommand",
"StageArtifactError",
]

View File

@ -0,0 +1,5 @@
from .cli import main
if __name__ == "__main__":
raise SystemExit(main())

1023
vendor/gs_playground/real2sim/builtin.py vendored Normal file

File diff suppressed because it is too large Load Diff

72
vendor/gs_playground/real2sim/camera.py vendored Normal file
View File

@ -0,0 +1,72 @@
"""Camera utilities shared by Real2Sim packaging and viewers."""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import numpy as np
from .errors import StageArtifactError
@dataclass(frozen=True)
class SourceCamera:
position: np.ndarray
xmat: np.ndarray
fovy_degrees: float
width: int
height: int
y_up: bool = False
def load_anysplat_source_camera(path: Path) -> SourceCamera:
with path.open("r", encoding="utf-8") as f:
payload = json.load(f)
if not isinstance(payload, dict):
raise StageArtifactError(f"camera JSON must be an object: {path}")
intrinsics = payload.get("intrinsics")
if not isinstance(intrinsics, dict):
raise StageArtifactError(f"camera JSON missing intrinsics: {path}")
for key in ("width", "height", "fy"):
if key not in intrinsics:
raise StageArtifactError(f"camera JSON intrinsics missing {key}: {path}")
extrinsic_value: Any = payload.get("pred_context_pose", {}).get("extrinsic")
if extrinsic_value is None:
extrinsic_value = payload.get("extrinsic")
extrinsic = _first_matrix(extrinsic_value, rows=4, cols=4, path=path)
c2w = np.linalg.inv(extrinsic).astype(np.float32)
width = int(intrinsics["width"])
height = int(intrinsics["height"])
fy = float(intrinsics["fy"])
if width <= 0 or height <= 0 or fy <= 0:
raise StageArtifactError(f"camera JSON has invalid dimensions or focal length: {path}")
fovy = math.degrees(2.0 * math.atan(height / (2.0 * fy)))
return SourceCamera(
position=c2w[:3, 3].astype(np.float32),
xmat=c2w[:3, :3].reshape(9).astype(np.float32),
fovy_degrees=fovy,
width=width,
height=height,
y_up=False,
)
def _first_matrix(value: Any, *, rows: int, cols: int, path: Path) -> np.ndarray:
if value is None:
raise StageArtifactError(f"camera JSON missing extrinsic matrix: {path}")
arr = np.asarray(value, dtype=np.float32)
while arr.ndim > 2 and arr.shape[0] == 1:
arr = arr[0]
if arr.ndim == 3 and arr.shape[1:] == (rows, cols):
arr = arr[0]
if arr.shape != (rows, cols):
raise StageArtifactError(f"camera JSON extrinsic must have shape {(rows, cols)}, got {arr.shape}: {path}")
return arr

489
vendor/gs_playground/real2sim/cli.py vendored Normal file
View File

@ -0,0 +1,489 @@
"""Command line interface for GS-Playground Real2Sim."""
from __future__ import annotations
import argparse
from pathlib import Path
from .contracts import write_json
from .paper_strict import check_paper_strict, format_report, validate_paper_strict
from .pipeline import Real2SimPipeline
from .pruning import PruneOptions
REPO_ROOT = Path(__file__).resolve().parents[1]
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="python -m real2sim", description="GS-Playground Real2Sim pipeline")
subparsers = parser.add_subparsers(dest="command", required=True)
run_parser = subparsers.add_parser("run", help="Run the full image/video-to-sim-ready-scene pipeline")
run_parser.add_argument("--input", type=Path, required=True, help="Input RGB image or video")
run_parser.add_argument("--output", type=Path, required=True, help="Output package directory")
run_parser.add_argument("--work-dir", type=Path, default=None, help="Intermediate artifact directory")
run_parser.add_argument("--prompt", default="all task-relevant objects", help="Open-vocabulary detection prompt")
run_parser.add_argument("--config", type=Path, default=None, help="Real2Sim stage command config JSON")
run_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"], help="Demo robot config to emit")
run_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"], help="Real2Sim backend")
run_parser.add_argument("--dynamic-objects", action="store_true", help="Emit free joints for reconstructed objects")
run_parser.add_argument("--resume", action="store_true", help="Reuse existing work-dir artifacts")
run_parser.add_argument("--dry-run", action="store_true", help="Print/validate configured commands without requiring outputs")
run_parser.add_argument("--paper-strict", action="store_true", help="Require exact paper-equivalent external stages and runtime prerequisites")
run_parser.add_argument("--synthetic-multiview", action="store_true", help="Expand a single input image into synthetic multiview frames before scene reconstruction")
run_parser.add_argument(
"--synthetic-multiview-mode",
default="object",
choices=["object", "scene"],
help="Prompt style for synthetic multiview expansion: object turntable or scene completion",
)
run_parser.add_argument(
"--pipeline-mode",
default="full",
choices=["full", "scene-only", "object-only"],
help="Pipeline scope: full (scene+objects), scene-only, or object-only",
)
run_parser.add_argument("--skip-scene-reconstruction", action="store_true", help="Skip scene reconstruction in an existing work-dir run")
run_parser.add_argument(
"--skip-object-reconstruction",
action="store_true",
help="Skip object reconstruction and keep only the reconstructed scene",
)
run_parser.add_argument("--no-prune", action="store_true", help="Skip 3DGS pruning")
run_parser.add_argument("--keep-ratio", type=float, default=None, help="Pruning keep ratio")
run_parser.set_defaults(func=_run)
scene_parser = subparsers.add_parser("generate-scene", help="Generate a scene-only Real2Sim package")
scene_parser.add_argument("--input", type=Path, required=True, help="Input RGB image, image directory, or video")
scene_parser.add_argument("--output", type=Path, required=True, help="Output scene package directory")
scene_parser.add_argument("--work-dir", type=Path, default=None, help="Intermediate artifact directory")
scene_parser.add_argument("--prompt", default="static navigable scene", help="Scene reconstruction prompt/context")
scene_parser.add_argument("--config", type=Path, default=None, help="Real2Sim stage command config JSON")
scene_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"], help="Demo robot config to emit")
scene_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"], help="Scene generation backend")
scene_parser.add_argument("--dynamic-objects", action="store_true", help=argparse.SUPPRESS)
scene_parser.add_argument("--resume", action="store_true", help="Reuse existing work-dir artifacts")
scene_parser.add_argument("--dry-run", action="store_true", help="Print/validate configured commands without requiring outputs")
scene_parser.add_argument("--paper-strict", action="store_true", help="Require exact paper-equivalent external scene stages")
scene_parser.add_argument("--synthetic-multiview", action="store_true", help="Expand a single input image into synthetic multiview frames before scene reconstruction")
scene_parser.add_argument(
"--scene-completion",
action="store_true",
help="Alias for scene-style synthetic multiview completion when a single image has insufficient scene evidence",
)
scene_parser.add_argument("--no-prune", action="store_true", help="Skip 3DGS pruning")
scene_parser.add_argument("--keep-ratio", type=float, default=None, help="Pruning keep ratio")
scene_parser.set_defaults(func=_generate_scene)
package_parser = subparsers.add_parser("package-existing", help="Package existing 3DGS/mesh assets as a GS-Playground scene")
package_parser.add_argument("--source", type=Path, required=True, help="Original capture or source path")
package_parser.add_argument("--scene-ply", type=Path, required=True, help="Scene/background 3DGS PLY")
package_parser.add_argument("--scene-mesh", type=Path, default=None, help="Optional scene mesh OBJ")
package_parser.add_argument("--objects-json", type=Path, default=None, help="Optional object asset list JSON")
package_parser.add_argument("--cameras-json", type=Path, default=None, help="Optional AnySplat camera intrinsics/extrinsics JSON")
package_parser.add_argument("--depth-map", type=Path, default=None, help="Optional monocular depth artifact used for alignment")
package_parser.add_argument("--background-image", type=Path, default=None, help="Optional inpainted background image artifact")
package_parser.add_argument("--preview-image", type=Path, default=None, help="Optional source-view 3DGS preview image artifact")
package_parser.add_argument("--mesh-cleanup-report", type=Path, default=None, help="Optional scene mesh cleanup report JSON")
package_parser.add_argument("--texture-bake-report", type=Path, default=None, help="Optional COLMAP texture bake report JSON")
package_parser.add_argument("--scene-structure-report", type=Path, default=None, help="Optional scene structure report JSON")
package_parser.add_argument("--output", type=Path, required=True, help="Output package directory")
package_parser.add_argument("--prompt", default="Navigate in the reconstructed Real2Sim scene")
package_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"])
package_parser.add_argument("--dynamic-objects", action="store_true")
package_parser.set_defaults(func=_package_existing)
check_parser = subparsers.add_parser("check", help="Check configured external model commands and local packages")
check_parser.add_argument("--config", type=Path, default=None)
check_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"])
check_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"])
check_parser.add_argument("--dynamic-objects", action="store_true")
check_parser.add_argument("--paper-strict", action="store_true", help="Validate exact paper-equivalent runtime prerequisites")
check_parser.set_defaults(func=_check)
example_parser = subparsers.add_parser("write-example-config", help="Write a config template for external model commands")
example_parser.add_argument("--output", type=Path, required=True)
example_parser.set_defaults(func=_write_example_config)
args = parser.parse_args(argv)
return int(args.func(args))
def _pipeline(args) -> Real2SimPipeline:
return Real2SimPipeline.from_config_file(
repo_root=REPO_ROOT,
config_file=getattr(args, "config", None),
robot=args.robot,
dynamic_objects=args.dynamic_objects,
dry_run=getattr(args, "dry_run", False),
backend="external" if getattr(args, "paper_strict", False) else getattr(args, "backend", "auto"),
)
def _run(args) -> int:
mode = str(getattr(args, "pipeline_mode", "full"))
skip_scene = bool(getattr(args, "skip_scene_reconstruction", False))
skip_objects = bool(getattr(args, "skip_object_reconstruction", False))
if mode != "full" and (skip_scene or skip_objects):
raise SystemExit("cannot combine --pipeline-mode with --skip-scene-reconstruction/--skip-object-reconstruction")
enable_scene_reconstruction = True
enable_object_reconstruction = True
if mode == "scene-only":
enable_scene_reconstruction = True
enable_object_reconstruction = False
elif mode == "object-only":
enable_scene_reconstruction = False
enable_object_reconstruction = True
else:
enable_scene_reconstruction = not skip_scene
enable_object_reconstruction = not skip_objects
if not enable_scene_reconstruction and not enable_object_reconstruction:
raise SystemExit("both scene and object reconstruction are disabled")
resume = args.resume
if mode == "object-only":
resume = True
pipeline = _pipeline(args)
paper_strict = bool(getattr(args, "paper_strict", False))
if paper_strict and (args.no_prune or args.keep_ratio is not None):
raise SystemExit("--paper-strict cannot be combined with --no-prune or --keep-ratio; use pruning.mode=speedy_splat")
if paper_strict:
report = check_paper_strict(pipeline.config, REPO_ROOT)
for line in format_report(report):
print(line)
if args.dry_run:
return 0 if report["ok"] else 1
validate_paper_strict(pipeline.config, REPO_ROOT)
if args.dry_run:
status = pipeline.check()
required = {"grounding_dino", "sam2", "instance_refinement", "lama", "anysplat", "sam3d", "pose_alignment"}
if args.backend in {"auto", "builtin", "hybrid"} and status["builtin_ready"]:
print("builtin: ready (numpy)")
if args.backend == "builtin":
return 0
for stage, info in status["stages"].items():
state = "configured" if info["configured"] else "missing"
print(f"{stage}: {state} ({info['env']})")
external_ready = all(status["stages"][stage]["configured"] for stage in required)
return 0 if external_ready or (args.backend in {"auto", "hybrid"} and status["builtin_ready"]) else 1
prune = None
if args.no_prune or args.keep_ratio is not None:
keep_ratio = args.keep_ratio if args.keep_ratio is not None else 0.3
prune = PruneOptions(enabled=not args.no_prune, keep_ratio=keep_ratio)
result = pipeline.run(
input_path=args.input,
output_dir=args.output,
work_dir=args.work_dir,
prompt=args.prompt,
resume=resume,
prune_options=prune,
enable_scene_reconstruction=enable_scene_reconstruction,
enable_object_reconstruction=enable_object_reconstruction,
synthetic_multiview=args.synthetic_multiview,
synthetic_multiview_mode=getattr(args, "synthetic_multiview_mode", "object"),
)
print(f"Real2Sim package: {result.package.root}")
print(f"Scene XML: {result.package.scene_xml}")
print(f"Demo config: {result.package.config_json}")
print(f"Manifest: {result.package.manifest_json}")
return 0
def _generate_scene(args) -> int:
pipeline = _pipeline(args)
paper_strict = bool(getattr(args, "paper_strict", False))
if paper_strict and (args.no_prune or args.keep_ratio is not None):
raise SystemExit("--paper-strict cannot be combined with --no-prune or --keep-ratio; use pruning.mode=speedy_splat")
if paper_strict:
report = check_paper_strict(pipeline.config, REPO_ROOT)
for line in format_report(report):
print(line)
if args.dry_run:
return 0 if report["ok"] else 1
validate_paper_strict(pipeline.config, REPO_ROOT)
if args.dry_run:
status = pipeline.check()
if args.backend in {"auto", "builtin", "hybrid"} and status["builtin_ready"]:
print("builtin: ready (numpy)")
return 0 if args.backend == "builtin" else 0
required = {"grounding_dino", "sam2", "instance_refinement", "lama", "anysplat"}
for stage, info in status["stages"].items():
state = "configured" if info["configured"] else "missing"
print(f"{stage}: {state} ({info['env']})")
return 0 if all(status["stages"][stage]["configured"] for stage in required) else 1
prune = None
if args.no_prune or args.keep_ratio is not None:
keep_ratio = args.keep_ratio if args.keep_ratio is not None else 0.3
prune = PruneOptions(enabled=not args.no_prune, keep_ratio=keep_ratio)
result = pipeline.run(
input_path=args.input,
output_dir=args.output,
work_dir=args.work_dir,
prompt=args.prompt,
resume=args.resume,
prune_options=prune,
enable_scene_reconstruction=True,
enable_object_reconstruction=False,
synthetic_multiview=args.synthetic_multiview or getattr(args, "scene_completion", False),
synthetic_multiview_mode="scene",
)
print(f"Real2Sim scene package: {result.package.root}")
print(f"Scene PLY: {result.manifest.scene.gaussian_ply}")
print(f"Scene XML: {result.package.scene_xml}")
print(f"Demo config: {result.package.config_json}")
print(f"Manifest: {result.package.manifest_json}")
return 0
def _package_existing(args) -> int:
result = _pipeline(args).package_existing(
source=args.source,
scene_ply=args.scene_ply,
scene_mesh=args.scene_mesh,
objects_json=args.objects_json,
output_dir=args.output,
prompt=args.prompt,
cameras_json=args.cameras_json,
depth_map=args.depth_map,
background_image=args.background_image,
preview_image=args.preview_image,
mesh_cleanup_report=args.mesh_cleanup_report,
texture_bake_report=args.texture_bake_report,
scene_structure_report=args.scene_structure_report,
)
print(f"Real2Sim package: {result.root}")
print(f"Scene XML: {result.scene_xml}")
print(f"Demo config: {result.config_json}")
print(f"Manifest: {result.manifest_json}")
return 0
def _check(args) -> int:
status = _pipeline(args).check()
paper_strict = bool(getattr(args, "paper_strict", False))
if paper_strict:
status["paper_strict"] = check_paper_strict(_pipeline(args).config, REPO_ROOT)
write_json(Path("/tmp/gsplayground_real2sim_check.json"), status)
for stage, info in status["stages"].items():
state = "configured" if info["configured"] else "missing"
print(f"{stage}: {state} ({info['env']})")
for module, ok in status["python_packages"].items():
prefix = "controller_python." if paper_strict else ""
print(f"{prefix}{module}: {'ok' if ok else 'missing'}")
if paper_strict:
for module in status["paper_strict"].get("python_modules", []):
stage = module.get("stage", "unknown")
name = module.get("name", "unknown")
state = "ok" if module.get("ok") else "missing"
print(f"paper_strict.{stage}.{name}: {state}")
for line in format_report(status["paper_strict"]):
print(line)
print("Wrote /tmp/gsplayground_real2sim_check.json")
return 0 if not paper_strict or status["paper_strict"]["ok"] else 1
def _write_example_config(args) -> int:
write_json(args.output, example_config())
print(f"Wrote {args.output}")
return 0
def example_config() -> dict:
return {
"stages": {
"synthetic_multiview": {
"command": [
"{python}",
"/path/to/synthetic_multiview_expand.py",
"--image",
"{image}",
"--output-dir",
"{synthetic_multiview_dir}",
"--frames-dir",
"{synthetic_frames_dir}",
"--prompt",
"{synthetic_multiview_prompt}",
"--size",
"1536x1024",
]
},
"grounding_dino": {
"command": [
"{python}",
"/path/to/grounding_dino_detect.py",
"--image",
"{image}",
"--prompt",
"{prompt}",
"--output",
"{detections_json}",
]
},
"sam2": {
"command": [
"{python}",
"/path/to/sam2_segment.py",
"--image",
"{image}",
"--detections",
"{detections_json}",
"--masks-dir",
"{masks_dir}",
"--output",
"{instances_json}",
]
},
"instance_refinement": {
"command": [
"{python}",
"/path/to/instance_refine.py",
"--instances",
"{instances_json}",
"--output",
"{instances_json}",
]
},
"lama": {
"command": [
"{python}",
"/path/to/lama_inpaint.py",
"--image",
"{image}",
"--instances",
"{instances_json}",
"--output",
"{background_image}",
"--prompt",
"{prompt}",
"--work-dir",
"{work_dir}",
"--redetect-command-env",
"REAL2SIM_GROUNDING_DINO_COMMAND",
"--segment-command-env",
"REAL2SIM_SAM2_COMMAND",
"--require-redetect-loop",
]
},
"depth_estimation": {
"command": [
"{python}",
"/path/to/depth_anything_depth.py",
"--image",
"{background_image}",
"--output",
"{depth_map}",
"--model",
"depth-anything/Depth-Anything-V2-Small-hf",
]
},
"anysplat": {
"command": [
"{python}",
"/path/to/anysplat_scene.py",
"--image",
"{background_image}",
"--frames-dir",
"{frames_dir}",
"--output-dir",
"{scene_reconstruction_dir}",
"--ply",
"{scene_ply}",
"--mesh",
"{scene_mesh}",
"--cameras",
"{cameras_json}",
"--depth",
"{depth_map}",
"--max-views",
"8",
]
},
"sam3d": {
"command": [
"{python}",
"/path/to/sam3d_object.py",
"--image",
"{image}",
"--mask",
"{mask}",
"--instance",
"{instance_json}",
"--output-dir",
"{object_dir}",
"--ply",
"{object_ply}",
"--mesh",
"{object_mesh}",
"--pose",
"{object_pose}",
"--cameras",
"{cameras_json}",
"--match-camera-preprocess",
"--object-centric-preprocess",
"--object-preprocess-size",
"896",
"--object-crop-margin",
"0.12",
]
},
"pose_alignment": {
"command": [
"{python}",
"/path/to/refine_pose_alignment.py",
"--assets",
"{aligned_assets_json}",
"--instances",
"{instances_json}",
"--depth",
"{depth_map}",
"--cameras",
"{cameras_json}",
"--work-dir",
"{work_dir}",
"--output",
"{aligned_assets_json}",
]
},
},
"builtin": {
"max_scene_points": 180000,
"max_object_points": 50000,
"max_objects": 10,
"min_component_area_ratio": 0.003,
"stage_min_component_area_ratio": 0.0005,
"focal_scale": 0.9,
"external_depth_weight": 0.65,
"color_enhance_strength": 0.35,
"scene_point_scale": 0.018,
"object_point_scale": 0.014,
"opacity_logit": 4.5,
"mask_close_passes": 2,
"mask_dilate_passes": 1,
"max_mask_overlap": 0.6,
},
"pruning": {
"enabled": True,
"keep_ratio": 0.85,
"mode": "speedy_splat",
"external_command": [
"{python}",
"scripts/real2sim_external/speedy_splat_prune.py",
"--input",
"{input}",
"--output",
"{output}",
"--image",
"{source_image}",
"--cameras",
"{cameras_json}",
"--keep-ratio",
"{keep_ratio}",
],
},
}

View File

@ -0,0 +1,97 @@
"""External command execution for Real2Sim stages.
The paper pipeline relies on large model systems that are not vendored in this
preview repository. This module keeps those integrations explicit: each stage
declares the command it needs and the artifacts it must produce.
"""
from __future__ import annotations
import os
import shlex
import subprocess
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Union
from .errors import MissingStageCommand, PipelineError
CommandValue = Union[str, Sequence[str]]
@dataclass(frozen=True)
class StageCommand:
name: str
env_var: str
description: str
required: bool = True
class _SafeFormatDict(dict[str, str]):
def __missing__(self, key: str) -> str:
raise KeyError(f"unknown command placeholder: {key}")
def _format_arg(value: str, context: Mapping[str, Any]) -> str:
flattened = {key: str(val) for key, val in context.items()}
return value.format_map(_SafeFormatDict(flattened))
def resolve_command(stage: StageCommand, config: Mapping[str, Any]) -> CommandValue | None:
stages = config.get("stages", {})
stage_config = stages.get(stage.name, {}) if isinstance(stages, Mapping) else {}
configured = stage_config.get("command") if isinstance(stage_config, Mapping) else None
env_command = os.environ.get(stage.env_var)
command = command_preferring_env(configured, env_command)
if command:
return command
if stage.required:
raise MissingStageCommand(
f"{stage.name} is required but no command was configured. "
f"Set stages.{stage.name}.command in the Real2Sim config or export {stage.env_var}. "
f"Expected stage: {stage.description}"
)
return None
def command_preferring_env(configured: CommandValue | None, env_command: str | None) -> CommandValue | None:
if env_command:
return env_command
return configured
def command_contains_placeholder(command: Any) -> bool:
if command is None:
return False
parts = command if isinstance(command, (list, tuple)) else [str(command)]
return any("/path/to/" in str(part) for part in parts)
def format_command(command: CommandValue, context: Mapping[str, Any]) -> list[str]:
if isinstance(command, str):
return [_format_arg(arg, context) for arg in shlex.split(command)]
return [_format_arg(str(arg), context) for arg in command]
def run_command(
stage: StageCommand,
command: CommandValue,
context: Mapping[str, Any],
*,
cwd: Path,
dry_run: bool = False,
) -> list[str]:
argv = format_command(command, context)
if not argv:
raise PipelineError(f"{stage.name} command is empty")
if dry_run:
return argv
completed = subprocess.run(argv, cwd=cwd, check=False)
if completed.returncode != 0:
raise PipelineError(f"{stage.name} command failed with exit code {completed.returncode}: {argv}")
return argv

View File

@ -0,0 +1,185 @@
"""Artifact contracts for Real2Sim stage boundaries."""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from .errors import StageArtifactError
def read_json(path: Path) -> Any:
if not path.exists():
raise StageArtifactError(f"missing JSON artifact: {path}")
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.write("\n")
def require_file(path: Path, description: str) -> Path:
if not path.is_file():
raise StageArtifactError(f"missing {description}: {path}")
return path
def require_dir(path: Path, description: str) -> Path:
if not path.is_dir():
raise StageArtifactError(f"missing {description}: {path}")
return path
@dataclass(frozen=True)
class Pose:
position: tuple[float, float, float] = (0.0, 0.0, 0.0)
quat_wxyz: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0)
scale: tuple[float, float, float] = (1.0, 1.0, 1.0)
@classmethod
def from_json(cls, path: Path) -> "Pose":
data = read_json(path)
if not isinstance(data, Mapping):
raise StageArtifactError(f"pose artifact must be a JSON object: {path}")
pos = data.get("position", data.get("translation", [0.0, 0.0, 0.0]))
quat = data.get("quat_wxyz", data.get("quaternion_wxyz"))
if quat is None and "quaternion_xyzw" in data:
qx, qy, qz, qw = _float_list(data["quaternion_xyzw"], 4, "quaternion_xyzw", path)
quat = [qw, qx, qy, qz]
if quat is None:
quat = [1.0, 0.0, 0.0, 0.0]
scale_raw = data.get("scale", [1.0, 1.0, 1.0])
if isinstance(scale_raw, (int, float)):
scale = (float(scale_raw), float(scale_raw), float(scale_raw))
else:
scale = tuple(_float_list(scale_raw, 3, "scale", path))
return cls(
position=tuple(_float_list(pos, 3, "position", path)),
quat_wxyz=tuple(_float_list(quat, 4, "quat_wxyz", path)),
scale=scale,
)
def to_json(self) -> dict[str, list[float]]:
return {
"position": list(self.position),
"quat_wxyz": list(self.quat_wxyz),
"scale": list(self.scale),
}
@dataclass(frozen=True)
class Instance:
instance_id: str
label: str
score: float | None = None
bbox_xyxy: tuple[float, float, float, float] | None = None
mask_path: Path | None = None
@classmethod
def from_mapping(cls, raw: Mapping[str, Any], base_dir: Path) -> "Instance":
raw_id = raw.get("id", raw.get("instance_id", raw.get("name")))
label = raw.get("label", raw.get("class", raw_id))
if raw_id is None or label is None:
raise StageArtifactError(f"instance entry requires id/name and label: {raw}")
bbox = raw.get("bbox_xyxy", raw.get("bbox"))
mask = raw.get("mask_path", raw.get("mask"))
return cls(
instance_id=str(raw_id),
label=str(label),
score=float(raw["score"]) if raw.get("score") is not None else None,
bbox_xyxy=tuple(_float_list(bbox, 4, "bbox", base_dir)) if bbox is not None else None,
mask_path=(base_dir / str(mask)).resolve() if mask else None,
)
@dataclass(frozen=True)
class Real2SimAsset:
name: str
label: str
gaussian_ply: Path
mesh_obj: Path | None = None
pose: Pose = field(default_factory=Pose)
dynamic: bool = False
source_instance_id: str | None = None
def to_json(self, root: Path) -> dict[str, Any]:
return {
"name": self.name,
"label": self.label,
"gaussian_ply": _relative_or_absolute(self.gaussian_ply, root),
"mesh_obj": _relative_or_absolute(self.mesh_obj, root) if self.mesh_obj else None,
"pose": self.pose.to_json(),
"dynamic": self.dynamic,
"source_instance_id": self.source_instance_id,
}
@dataclass(frozen=True)
class Real2SimManifest:
source: Path
prompt: str
scene: Real2SimAsset
objects: tuple[Real2SimAsset, ...] = ()
camera_artifacts: dict[str, Path] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
def to_json(self, root: Path) -> dict[str, Any]:
return {
"source": _relative_or_absolute(self.source, root),
"prompt": self.prompt,
"scene": self.scene.to_json(root),
"objects": [obj.to_json(root) for obj in self.objects],
"camera_artifacts": {
key: _relative_or_absolute(value, root)
for key, value in sorted(self.camera_artifacts.items())
},
"metadata": self.metadata,
}
def load_instances(path: Path) -> list[Instance]:
data = read_json(path)
entries = data.get("instances", data) if isinstance(data, Mapping) else data
if not isinstance(entries, list):
raise StageArtifactError(f"instances artifact must be a list or object with instances list: {path}")
return [Instance.from_mapping(entry, path.parent) for entry in entries]
def validate_detections(path: Path) -> None:
data = read_json(path)
entries = data.get("detections", data) if isinstance(data, Mapping) else data
if not isinstance(entries, list):
raise StageArtifactError(f"detections artifact must be a list or object with detections list: {path}")
for entry in entries:
if not isinstance(entry, Mapping):
raise StageArtifactError(f"detection entries must be JSON objects: {path}")
if "bbox" not in entry and "bbox_xyxy" not in entry:
raise StageArtifactError(f"detection entry missing bbox/bbox_xyxy: {entry}")
def _float_list(value: Any, size: int, field_name: str, path: Path) -> list[float]:
if not isinstance(value, Iterable) or isinstance(value, (str, bytes)):
raise StageArtifactError(f"{field_name} must be a list with {size} values in {path}")
vals = [float(item) for item in value]
if len(vals) != size:
raise StageArtifactError(f"{field_name} must have {size} values in {path}, got {len(vals)}")
return vals
def _relative_or_absolute(path: Path | None, root: Path) -> str | None:
if path is None:
return None
try:
return path.resolve().relative_to(root.resolve()).as_posix()
except ValueError:
return path.resolve().as_posix()

15
vendor/gs_playground/real2sim/errors.py vendored Normal file
View File

@ -0,0 +1,15 @@
"""Pipeline-specific exceptions."""
from __future__ import annotations
class PipelineError(RuntimeError):
"""Base class for Real2Sim pipeline failures."""
class MissingStageCommand(PipelineError):
"""Raised when a required external model stage is not configured."""
class StageArtifactError(PipelineError):
"""Raised when a stage does not produce the expected artifacts."""

View File

@ -0,0 +1,140 @@
"""Small image I/O helpers used by the built-in Real2Sim baseline."""
from __future__ import annotations
from pathlib import Path
import numpy as np
from .errors import PipelineError
def read_rgb(path: Path) -> np.ndarray:
suffix = path.suffix.lower()
if suffix == ".ppm":
return _read_ppm(path)
try:
import matplotlib.image as mpimg
except Exception as exc: # pragma: no cover - depends on runtime env
raise PipelineError(
f"reading {suffix or 'this image'} requires matplotlib in this environment. "
"Use a .ppm input or run through the project uv environment."
) from exc
arr = np.asarray(mpimg.imread(path), dtype=np.float32)
if arr.ndim == 2:
arr = np.repeat(arr[:, :, None], 3, axis=2)
if arr.shape[-1] == 4:
alpha = arr[:, :, 3:4]
arr = arr[:, :, :3] * alpha + (1.0 - alpha)
else:
arr = arr[:, :, :3]
if arr.dtype.kind in {"u", "i"} or arr.max() > 1.5:
arr = arr / 255.0
return np.clip(arr, 0.0, 1.0).astype(np.float32)
def read_gray(path: Path) -> np.ndarray:
suffix = path.suffix.lower()
if suffix == ".pgm":
return _read_pgm(path)
try:
import matplotlib.image as mpimg
except Exception as exc: # pragma: no cover - depends on runtime env
raise PipelineError(
f"reading {suffix or 'this mask'} requires matplotlib in this environment. "
"Use a .pgm mask or run through the project uv environment."
) from exc
arr = np.asarray(mpimg.imread(path), dtype=np.float32)
if arr.ndim == 3:
arr = arr[:, :, :3].mean(axis=2)
if arr.dtype.kind in {"u", "i"} or arr.max() > 1.5:
arr = arr / 255.0
return np.clip(arr, 0.0, 1.0).astype(np.float32)
def write_rgb(path: Path, image: np.ndarray) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
arr = np.clip(image, 0.0, 1.0)
if path.suffix.lower() == ".ppm":
_write_ppm(path, arr)
return
try:
import matplotlib.image as mpimg
mpimg.imsave(path, arr)
except Exception:
_write_ppm(path, arr)
def write_gray(path: Path, image: np.ndarray) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
arr = np.clip(image, 0.0, 1.0)
if path.suffix.lower() == ".pgm":
_write_pgm(path, arr)
return
try:
import matplotlib.image as mpimg
mpimg.imsave(path, arr, cmap="gray")
except Exception:
_write_pgm(path, arr)
def _read_ppm(path: Path) -> np.ndarray:
with path.open("rb") as f:
magic = f.readline().strip()
if magic != b"P6":
raise PipelineError(f"only binary P6 PPM input is supported without matplotlib: {path}")
dims = _next_ppm_token_line(f)
width, height = [int(x) for x in dims.split()]
maxval = int(_next_ppm_token_line(f))
if maxval <= 0 or maxval > 255:
raise PipelineError(f"unsupported PPM max value {maxval}: {path}")
data = np.frombuffer(f.read(width * height * 3), dtype=np.uint8)
if data.size != width * height * 3:
raise PipelineError(f"truncated PPM input: {path}")
return data.reshape(height, width, 3).astype(np.float32) / float(maxval)
def _read_pgm(path: Path) -> np.ndarray:
with path.open("rb") as f:
magic = f.readline().strip()
if magic != b"P5":
raise PipelineError(f"only binary P5 PGM input is supported without matplotlib: {path}")
dims = _next_ppm_token_line(f)
width, height = [int(x) for x in dims.split()]
maxval = int(_next_ppm_token_line(f))
if maxval <= 0 or maxval > 255:
raise PipelineError(f"unsupported PGM max value {maxval}: {path}")
data = np.frombuffer(f.read(width * height), dtype=np.uint8)
if data.size != width * height:
raise PipelineError(f"truncated PGM input: {path}")
return data.reshape(height, width).astype(np.float32) / float(maxval)
def _write_ppm(path: Path, image: np.ndarray) -> None:
arr = (np.clip(image, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8)
with path.open("wb") as f:
f.write(f"P6\n{arr.shape[1]} {arr.shape[0]}\n255\n".encode("ascii"))
f.write(arr[:, :, :3].tobytes())
def _write_pgm(path: Path, image: np.ndarray) -> None:
arr = (np.clip(image, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8)
with path.open("wb") as f:
f.write(f"P5\n{arr.shape[1]} {arr.shape[0]}\n255\n".encode("ascii"))
f.write(arr.tobytes())
def _next_ppm_token_line(f) -> str:
while True:
line = f.readline()
if not line:
raise PipelineError("unexpected EOF while reading PPM header")
line = line.strip()
if line and not line.startswith(b"#"):
return line.decode("ascii")

3145
vendor/gs_playground/real2sim/packaging.py vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,320 @@
"""Paper-strict Real2Sim runtime validation."""
from __future__ import annotations
import importlib.util as importlib_util
import os
import shlex
import shutil
import subprocess
import sys
from collections.abc import Mapping
from pathlib import Path
from typing import Any
from .commands import command_contains_placeholder, command_preferring_env
from .errors import PipelineError
REQUIRED_STAGES = (
"grounding_dino",
"sam2",
"instance_refinement",
"lama",
"anysplat",
"sam3d",
"pose_alignment",
)
MIN_SAM3D_VRAM_MIB = 32768
REQUIRED_STAGE_PYTHON_MODULES = {
"grounding_dino": ("transformers",),
"sam2": ("sam2",),
"instance_refinement": ("numpy", "PIL"),
"lama": ("simple_lama_inpainting",),
"anysplat": ("torch", "pytorch3d", "gsplat", "plyfile", "src.model.model.anysplat", "src.model.ply_export"),
"sam3d": ("torch", "pytorch3d", "kaolin", "gsplat", "sam3d_objects", "inference"),
"pose_alignment": ("numpy", "PIL"),
"speedy_splat": ("gaussian_renderer",),
}
def validate_paper_strict(
config: Mapping[str, Any],
repo_root: Path,
*,
env: Mapping[str, str] | None = None,
gpu_memory_mib: int | None = None,
) -> dict[str, Any]:
report = check_paper_strict(config, repo_root, env=env, gpu_memory_mib=gpu_memory_mib)
if report["ok"]:
return report
summary = "; ".join(f"{failure['name']}: {failure['message']}" for failure in report["failures"])
raise PipelineError(f"paper-strict Real2Sim prerequisites are not satisfied: {summary}")
def check_paper_strict(
config: Mapping[str, Any],
repo_root: Path,
*,
env: Mapping[str, str] | None = None,
gpu_memory_mib: int | None = None,
) -> dict[str, Any]:
env_map = dict(os.environ if env is None else env)
failures: list[dict[str, str]] = []
checks: list[dict[str, Any]] = []
stages = config.get("stages", {}) if isinstance(config.get("stages", {}), Mapping) else {}
for stage in REQUIRED_STAGES:
command = configured_stage_command(stages, stage, env_map)
ok = bool(command)
checks.append({"name": f"stage_{stage}", "ok": ok})
if not ok:
failures.append({"name": f"stage_{stage}", "message": f"missing command for {stage}"})
elif command_contains_placeholder(command):
failures.append({"name": f"stage_{stage}", "message": f"command for {stage} still contains a placeholder path"})
else:
executable_error = command_executable_error(command)
if executable_error:
failures.append({"name": f"stage_{stage}", "message": executable_error})
pruning = config.get("pruning", {}) if isinstance(config.get("pruning", {}), Mapping) else {}
pruning_mode = str(pruning.get("mode", "")).replace("-", "_").lower()
speedy_command = command_preferring_env(pruning.get("external_command"), env_map.get("REAL2SIM_SPEEDY_SPLAT_COMMAND", ""))
if pruning_mode != "speedy_splat":
failures.append({"name": "speedy_splat_mode", "message": "pruning.mode must be speedy_splat"})
if not speedy_command:
failures.append({"name": "speedy_splat_command", "message": "missing Speedy-Splat external command"})
elif command_contains_placeholder(speedy_command):
failures.append({"name": "speedy_splat_command", "message": "Speedy-Splat command still contains a placeholder path"})
else:
executable_error = command_executable_error(speedy_command)
if executable_error:
failures.append({"name": "speedy_splat_command", "message": executable_error})
nested_grounding = configured_stage_command(stages, "grounding_dino", env_map)
nested_sam2 = configured_stage_command(stages, "sam2", env_map)
if not nested_grounding or not nested_sam2:
failures.append(
{
"name": "inpainting_redetect_loop",
"message": "strict LaMa recovery requires configured GroundingDINO and SAM2 commands",
}
)
anysplat_root = Path(env_map.get("REAL2SIM_ANYSPLAT_ROOT", ""))
require_path(failures, "anysplat_root", anysplat_root, "REAL2SIM_ANYSPLAT_ROOT")
if anysplat_root:
require_path(failures, "anysplat_api", anysplat_root / "src" / "model" / "model" / "anysplat.py", "AnySplat API")
require_path(failures, "anysplat_export", anysplat_root / "src" / "model" / "ply_export.py", "AnySplat export_ply")
sam3d_root = Path(env_map.get("REAL2SIM_SAM3D_ROOT", ""))
require_path(failures, "sam3d_root", sam3d_root, "REAL2SIM_SAM3D_ROOT")
if sam3d_root:
require_path(failures, "sam3d_inference", sam3d_root / "notebook" / "inference.py", "SAM-3D notebook inference API")
sam3d_config = Path(env_map.get("REAL2SIM_SAM3D_CONFIG", "")) if env_map.get("REAL2SIM_SAM3D_CONFIG") else sam3d_root / "checkpoints" / "hf" / "pipeline.yaml"
require_path(failures, "sam3d_config", sam3d_config, "SAM-3D checkpoint pipeline.yaml")
sam2_checkpoint = Path(env_map.get("REAL2SIM_SAM2_CHECKPOINT", ""))
require_path(failures, "sam2_checkpoint", sam2_checkpoint, "REAL2SIM_SAM2_CHECKPOINT")
speedy_root = Path(env_map.get("REAL2SIM_SPEEDY_SPLAT_ROOT", ""))
require_path(failures, "speedy_splat_root", speedy_root, "REAL2SIM_SPEEDY_SPLAT_ROOT")
module_checks = []
for stage, module_names in REQUIRED_STAGE_PYTHON_MODULES.items():
command = speedy_command if stage == "speedy_splat" else configured_stage_command(stages, stage, env_map)
if not command:
continue
if command_contains_placeholder(command) or command_executable_error(command):
continue
python_executable = python_executable_for_command(command)
extra_paths = stage_module_extra_paths(stage, env_map)
for module_name in module_names:
ok = module_available(module_name, python_executable, repo_root, env_map, extra_paths=extra_paths)
module_checks.append(
{
"stage": stage,
"name": module_name,
"ok": ok,
"python": python_executable,
"extra_paths": list(extra_paths),
}
)
if not ok:
failures.append(
{
"name": f"python_module_{module_name}",
"message": (
f"missing required Python module for paper-strict Real2Sim stage "
f"{stage}: {module_name} in {python_executable}"
),
}
)
detected_vram = detect_gpu_memory_mib() if gpu_memory_mib is None else gpu_memory_mib
if detected_vram is None:
failures.append({"name": "sam3d_vram", "message": "could not determine NVIDIA GPU memory"})
elif detected_vram < MIN_SAM3D_VRAM_MIB:
failures.append(
{
"name": "sam3d_vram",
"message": f"SAM-3D official setup requires at least {MIN_SAM3D_VRAM_MIB} MiB VRAM, found {detected_vram} MiB",
}
)
return {
"ok": not failures,
"checks": checks,
"python_modules": module_checks,
"failures": failures,
"gpu_memory_mib": detected_vram,
"required_stages": list(REQUIRED_STAGES),
}
def stage_env(stage: str) -> str:
return {
"grounding_dino": "REAL2SIM_GROUNDING_DINO_COMMAND",
"sam2": "REAL2SIM_SAM2_COMMAND",
"instance_refinement": "REAL2SIM_INSTANCE_REFINEMENT_COMMAND",
"lama": "REAL2SIM_LAMA_COMMAND",
"anysplat": "REAL2SIM_ANYSPLAT_COMMAND",
"sam3d": "REAL2SIM_SAM3D_COMMAND",
"pose_alignment": "REAL2SIM_POSE_ALIGNMENT_COMMAND",
}[stage]
def configured_stage_command(stages: Mapping[str, Any], stage: str, env_map: Mapping[str, str]) -> Any:
stage_config = stages.get(stage, {}) if isinstance(stages, Mapping) else {}
command = stage_config.get("command") if isinstance(stage_config, Mapping) else None
return command_preferring_env(command, env_map.get(stage_env(stage), ""))
def require_path(failures: list[dict[str, str]], name: str, path: Path, description: str) -> None:
if not str(path) or str(path) == "." or not path.exists():
failures.append({"name": name, "message": f"missing {description}: {path}"})
def command_executable_error(command: Any) -> str | None:
executable = command_executable(command)
if not executable or ("{" in executable and "}" in executable):
return None
if os.sep in executable or (os.altsep and os.altsep in executable):
path = Path(executable)
if not path.exists():
return f"command executable does not exist: {executable}"
if path.is_dir():
return f"command executable is a directory: {executable}"
if not os.access(path, os.X_OK):
return f"command executable is not executable: {executable}"
return None
if shutil.which(executable) is None:
return f"command executable not found on PATH: {executable}"
return None
def command_executable(command: Any) -> str | None:
if command is None:
return None
if isinstance(command, str):
try:
parts = shlex.split(command)
except ValueError:
return ""
elif isinstance(command, (list, tuple)):
parts = [str(part) for part in command]
else:
parts = [str(command)]
return parts[0] if parts else None
def python_executable_for_command(command: Any) -> str:
executable = command_executable(command)
if not executable or ("{" in executable and "}" in executable):
return sys.executable
return executable
def stage_module_extra_paths(stage: str, env_map: Mapping[str, str]) -> tuple[str, ...]:
paths: list[str] = []
if stage == "anysplat":
root = env_map.get("REAL2SIM_ANYSPLAT_ROOT", "")
if root:
paths.append(root)
elif stage == "sam3d":
root = env_map.get("REAL2SIM_SAM3D_ROOT", "")
if root:
paths.extend([(Path(root) / "notebook").as_posix(), root])
elif stage == "speedy_splat":
root = env_map.get("REAL2SIM_SPEEDY_SPLAT_ROOT", "")
if root:
paths.append(root)
return tuple(paths)
def module_available(
module_name: str,
python_executable: str,
repo_root: Path,
env_map: Mapping[str, str],
*,
extra_paths: tuple[str, ...] = (),
) -> bool:
if python_executable == sys.executable and not extra_paths and not env_map.get("PYTHONPATH"):
try:
return importlib_util.find_spec(module_name) is not None
except (ImportError, AttributeError, ValueError):
return False
module_env = dict(os.environ)
module_env.update(env_map)
module_env.setdefault("LIDRA_SKIP_INIT", "1")
pythonpath_parts = [path for path in extra_paths if path]
if module_env.get("PYTHONPATH"):
pythonpath_parts.append(module_env["PYTHONPATH"])
if pythonpath_parts:
module_env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts)
code = "import importlib, sys; importlib.import_module(sys.argv[1])"
try:
completed = subprocess.run(
[python_executable, "-c", code, module_name],
cwd=repo_root,
env=module_env,
check=False,
capture_output=True,
text=True,
timeout=30,
)
except (FileNotFoundError, NotADirectoryError, PermissionError, subprocess.TimeoutExpired):
return False
return completed.returncode == 0
def detect_gpu_memory_mib() -> int | None:
try:
completed = subprocess.run(
["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
check=False,
capture_output=True,
text=True,
)
except FileNotFoundError:
return None
if completed.returncode != 0:
return None
values = []
for line in completed.stdout.splitlines():
line = line.strip()
if not line:
continue
try:
values.append(int(float(line.split()[0])))
except ValueError:
continue
return max(values) if values else None
def format_report(report: Mapping[str, Any]) -> list[str]:
lines = [f"paper_strict: {'ok' if report.get('ok') else 'failed'}"]
for failure in report.get("failures", []):
lines.append(f"{failure['name']}: {failure['message']}")
return lines

2819
vendor/gs_playground/real2sim/pipeline.py vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,94 @@
"""3DGS pruning helpers."""
from __future__ import annotations
import shutil
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from collections.abc import Mapping
from typing import Any, Sequence, Union
from .commands import format_command
from .errors import PipelineError
CommandValue = Union[str, Sequence[str]]
@dataclass(frozen=True)
class PruneOptions:
enabled: bool = True
keep_ratio: float = 0.3
keep_count: int | None = None
mode: str = "opacity_area"
external_command: CommandValue | None = None
def prune_ply(
src: Path,
dst: Path,
options: PruneOptions,
repo_root: Path,
*,
extra_context: Mapping[str, Any] | None = None,
) -> Path:
dst.parent.mkdir(parents=True, exist_ok=True)
if not options.enabled or options.keep_ratio >= 1.0:
shutil.copy2(src, dst)
return dst
if options.mode.replace("-", "_").lower() in {"speedy_splat", "speedysplat"}:
return prune_with_speedy_splat(src, dst, options, repo_root, extra_context=extra_context)
script = repo_root / "benchmark" / "scripts" / "prune_gaussians.py"
if not script.is_file():
raise PipelineError(f"pruning script not found: {script}")
cmd = [sys.executable, script.as_posix(), "--input", src.as_posix(), "--output", dst.as_posix(), "--mode", options.mode]
if options.keep_count is not None:
cmd.extend(["--keep-count", str(options.keep_count)])
else:
cmd.extend(["--keep-ratio", str(options.keep_ratio)])
completed = subprocess.run(cmd, cwd=repo_root, check=False)
if completed.returncode != 0:
raise PipelineError(f"3DGS pruning failed with exit code {completed.returncode}: {cmd}")
return dst
def prune_with_speedy_splat(
src: Path,
dst: Path,
options: PruneOptions,
repo_root: Path,
*,
extra_context: Mapping[str, Any] | None = None,
) -> Path:
if options.external_command is None:
raise PipelineError(
"Speedy-Splat pruning requires pruning.external_command or an equivalent "
"configured command. The built-in opacity_area pruner is not paper-equivalent."
)
extras = dict(extra_context or {})
context = {
"input": src,
"output": dst,
"keep_ratio": options.keep_ratio,
"keep_count": options.keep_count or "",
"mode": options.mode,
"python": sys.executable,
"source": extras.get("source", ""),
"source_image": extras.get("source_image", extras.get("image", "")),
"image": extras.get("image", extras.get("source_image", "")),
"cameras_json": extras.get("cameras_json", extras.get("cameras", "")),
"cameras": extras.get("cameras", extras.get("cameras_json", "")),
}
argv = format_command(options.external_command, context)
completed = subprocess.run(argv, cwd=repo_root, check=False)
if completed.returncode != 0:
raise PipelineError(f"Speedy-Splat pruning failed with exit code {completed.returncode}: {argv}")
if not dst.exists():
raise PipelineError(f"Speedy-Splat command did not write output PLY: {dst}")
return dst

View File

@ -0,0 +1,281 @@
#!/usr/bin/env python3
"""Create a capture acceptance report for high-fidelity Real2Sim reconstruction."""
from __future__ import annotations
import argparse
import json
import math
from pathlib import Path
from typing import Any
def main() -> int:
parser = argparse.ArgumentParser(description="Evaluate whether a capture is ready for high-fidelity Real2Sim reconstruction")
parser.add_argument("--sfm-report", type=Path, required=True, help="COLMAP capture report JSON")
parser.add_argument("--output-json", type=Path, required=True)
parser.add_argument("--output-md", type=Path, required=True)
parser.add_argument("--min-frames", type=int, default=30)
parser.add_argument("--min-registered-ratio", type=float, default=0.70)
parser.add_argument("--min-registered-images", type=int, default=20)
parser.add_argument("--min-sparse-points", type=int, default=5000)
parser.add_argument("--min-verified-pairs", type=int, default=40)
parser.add_argument(
"--min-camera-path-length",
type=float,
default=1.0,
help="Minimum COLMAP camera path length for space-level capture acceptance",
)
parser.add_argument(
"--min-camera-planar-coverage-ratio",
type=float,
default=0.12,
help="Minimum second-largest camera extent / largest extent ratio; rejects nearly 1D camera tracks",
)
parser.add_argument(
"--max-camera-step-ratio",
type=float,
default=0.55,
help="Maximum largest camera step / full path length ratio; rejects discontinuous captures",
)
args = parser.parse_args()
sfm = json.loads(args.sfm_report.read_text(encoding="utf-8"))
report = build_acceptance_report(sfm, args)
args.output_json.parent.mkdir(parents=True, exist_ok=True)
args.output_md.parent.mkdir(parents=True, exist_ok=True)
args.output_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
args.output_md.write_text(render_markdown(report), encoding="utf-8")
return 0 if report["accepted"] else 2
def build_acceptance_report(sfm: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]:
stats = sfm.get("database_stats") if isinstance(sfm.get("database_stats"), dict) else {}
camera_metrics = camera_space_metrics(sfm)
metrics = {
"frame_count": int(sfm.get("frame_count", 0) or 0),
"registered_image_count": int(sfm.get("registered_image_count", 0) or 0),
"registered_ratio": float(sfm.get("registered_ratio", 0.0) or 0.0),
"sparse_point_count": int(sfm.get("sparse_point_count", 0) or 0),
"keypoint_total": int(stats.get("keypoint_total", 0) or 0),
"keypoint_mean": float(stats.get("keypoint_mean", 0.0) or 0.0),
"matched_pair_count": int(stats.get("matched_pair_count", 0) or 0),
"verified_pair_count": int(stats.get("verified_pair_count", 0) or 0),
"verified_pair_mean": float(stats.get("verified_pair_mean", 0.0) or 0.0),
**camera_metrics,
}
gates = {
"enough_frames": gate(metrics["frame_count"] >= args.min_frames, metrics["frame_count"], args.min_frames),
"enough_registered_images": gate(
metrics["registered_image_count"] >= args.min_registered_images,
metrics["registered_image_count"],
args.min_registered_images,
),
"registered_ratio": gate(
metrics["registered_ratio"] >= args.min_registered_ratio,
round(metrics["registered_ratio"], 4),
args.min_registered_ratio,
),
"sparse_points": gate(metrics["sparse_point_count"] >= args.min_sparse_points, metrics["sparse_point_count"], args.min_sparse_points),
"verified_pairs": gate(metrics["verified_pair_count"] >= args.min_verified_pairs, metrics["verified_pair_count"], args.min_verified_pairs),
"camera_path_length": gate(
metrics["camera_path_length"] >= args.min_camera_path_length,
round(metrics["camera_path_length"], 4),
args.min_camera_path_length,
),
"camera_planar_coverage": gate(
metrics["camera_planar_coverage_ratio"] >= args.min_camera_planar_coverage_ratio,
round(metrics["camera_planar_coverage_ratio"], 4),
args.min_camera_planar_coverage_ratio,
),
"camera_step_consistency": gate(
metrics["camera_max_step_ratio"] is not None and metrics["camera_max_step_ratio"] <= args.max_camera_step_ratio,
round(metrics["camera_max_step_ratio"], 4) if metrics["camera_max_step_ratio"] is not None else None,
args.max_camera_step_ratio,
),
}
accepted = all(item["passed"] for item in gates.values())
blockers = [name for name, item in gates.items() if not item["passed"]]
return {
"format": "gs-playground-real2sim-capture-acceptance-v1",
"accepted": accepted,
"target_quality": "polycam_realitycapture_like_space",
"decision": "ready_for_pose_initialized_reconstruction" if accepted else "recapture_required_before_high_fidelity_reconstruction",
"metrics": metrics,
"gates": gates,
"blockers": blockers,
"sfm_warnings": sfm.get("warnings", []),
"sfm_recommendations": sfm.get("recommendations", []),
"capture_requirements": capture_requirements(blockers, metrics),
}
def camera_space_metrics(sfm: dict[str, Any]) -> dict[str, Any]:
path = sfm.get("camera_path") if isinstance(sfm.get("camera_path"), dict) else {}
poses = camera_positions(sfm.get("camera_poses"))
if poses:
extents = axis_extents(poses)
path_length = path_length_from_positions(poses)
step_distances = [
distance(poses[index - 1], poses[index])
for index in range(1, len(poses))
]
closure_distance = distance(poses[0], poses[-1]) if len(poses) >= 2 else 0.0
else:
extents = numeric_list(path.get("bounds_extent"), expected=3) or [0.0, 0.0, 0.0]
path_length = float(path.get("path_length", 0.0) or 0.0)
step_distances = []
closure_distance = None
sorted_extents = sorted((max(float(value), 0.0) for value in extents), reverse=True)
dominant_extent = sorted_extents[0] if sorted_extents else 0.0
secondary_extent = sorted_extents[1] if len(sorted_extents) >= 2 else 0.0
planar_ratio = secondary_extent / dominant_extent if dominant_extent > 1e-9 else 0.0
max_step = max(step_distances) if step_distances else None
median_step = percentile(step_distances, 0.5) if step_distances else None
max_step_ratio = max_step / path_length if max_step is not None and path_length > 1e-9 else None
closure_ratio = closure_distance / path_length if closure_distance is not None and path_length > 1e-9 else None
return {
"camera_pose_count": int(path.get("pose_count", len(poses)) or len(poses)),
"camera_path_length": round(float(path_length), 6),
"camera_bounds_extent": round_list(extents),
"camera_dominant_extent": round(float(dominant_extent), 6),
"camera_secondary_extent": round(float(secondary_extent), 6),
"camera_planar_coverage_ratio": round(float(planar_ratio), 6),
"camera_max_step_distance": round(float(max_step), 6) if max_step is not None else None,
"camera_median_step_distance": round(float(median_step), 6) if median_step is not None else None,
"camera_max_step_ratio": round(float(max_step_ratio), 6) if max_step_ratio is not None else None,
"camera_loop_closure_distance": round(float(closure_distance), 6) if closure_distance is not None else None,
"camera_loop_closure_ratio": round(float(closure_ratio), 6) if closure_ratio is not None else None,
}
def camera_positions(raw_poses: Any) -> list[list[float]]:
positions: list[list[float]] = []
if not isinstance(raw_poses, list):
return positions
for pose in raw_poses:
if not isinstance(pose, dict):
continue
position = numeric_list(pose.get("position"), expected=3)
if position is not None:
positions.append(position)
return positions
def numeric_list(value: Any, *, expected: int) -> list[float] | None:
if not isinstance(value, list) or len(value) != expected:
return None
try:
values = [float(item) for item in value]
except (TypeError, ValueError):
return None
if not all(math.isfinite(item) for item in values):
return None
return values
def axis_extents(positions: list[list[float]]) -> list[float]:
return [
max(position[axis] for position in positions) - min(position[axis] for position in positions)
for axis in range(3)
]
def path_length_from_positions(positions: list[list[float]]) -> float:
return sum(distance(positions[index - 1], positions[index]) for index in range(1, len(positions)))
def distance(a: list[float], b: list[float]) -> float:
return math.sqrt(sum((a[index] - b[index]) ** 2 for index in range(3)))
def percentile(values: list[float], fraction: float) -> float:
if not values:
return 0.0
ordered = sorted(float(value) for value in values)
index = min(max(int(round((len(ordered) - 1) * fraction)), 0), len(ordered) - 1)
return ordered[index]
def round_list(values: list[float]) -> list[float]:
return [round(float(value), 6) for value in values]
def gate(passed: bool, value: Any, threshold: int | float) -> dict[str, Any]:
return {"passed": bool(passed), "value": value, "threshold": threshold}
def capture_requirements(blockers: list[str], metrics: dict[str, Any]) -> list[str]:
requirements = [
"Capture one room-sized volume with continuous overlap; do not jump between unrelated viewpoints.",
"Move slowly with strong sideways parallax around desks, cabinets, floor, and wall features.",
"Keep 30-80 sharp frames after blur filtering; avoid fast pans, glass-only regions, and blank walls.",
]
if "registered_ratio" in blockers or "enough_registered_images" in blockers:
requirements.append(
f"Current registration is {metrics['registered_image_count']} images / {metrics['frame_count']} frames; recapture until COLMAP registers at least 70 percent."
)
if "sparse_points" in blockers:
requirements.append(
f"Current sparse points are {metrics['sparse_point_count']}; add textured overlap and lateral baseline until sparse points exceed 5000."
)
if "verified_pairs" in blockers:
requirements.append(
f"Current verified pairs are {metrics['verified_pair_count']}; increase adjacent overlap and reduce motion blur."
)
if "camera_path_length" in blockers:
requirements.append(
f"Current registered camera path length is {metrics['camera_path_length']}; walk a longer continuous path with sideways parallax around the room volume."
)
if "camera_planar_coverage" in blockers:
requirements.append(
f"Current camera planar coverage ratio is {metrics['camera_planar_coverage_ratio']}; avoid a single straight-line sweep and capture from at least two lateral directions."
)
if "camera_step_consistency" in blockers:
requirements.append(
f"Current camera max-step ratio is {metrics['camera_max_step_ratio']}; remove discontinuous jumps or recapture as one continuous overlapping trajectory."
)
return requirements
def render_markdown(report: dict[str, Any]) -> str:
metrics = report["metrics"]
lines = [
"# Real2Sim Capture Acceptance Report",
"",
f"Decision: `{report['decision']}`",
f"Accepted: `{str(report['accepted']).lower()}`",
"",
"## Metrics",
"",
"| Metric | Value |",
"| --- | ---: |",
]
for key, value in metrics.items():
lines.append(f"| `{key}` | {value} |")
lines.extend(["", "## Gates", "", "| Gate | Value | Threshold | Passed |", "| --- | ---: | ---: | --- |"])
for name, item in report["gates"].items():
lines.append(f"| `{name}` | {item['value']} | {item['threshold']} | `{str(item['passed']).lower()}` |")
if report["blockers"]:
lines.extend(["", "## Blockers", ""])
for blocker in report["blockers"]:
lines.append(f"- `{blocker}`")
if report["capture_requirements"]:
lines.extend(["", "## Capture Requirements", ""])
for requirement in report["capture_requirements"]:
lines.append(f"- {requirement}")
if report["sfm_warnings"]:
lines.extend(["", "## SfM Warnings", ""])
for warning in report["sfm_warnings"]:
lines.append(f"- {warning}")
if report["sfm_recommendations"]:
lines.extend(["", "## SfM Recommendations", ""])
for recommendation in report["sfm_recommendations"]:
lines.append(f"- {recommendation}")
lines.append("")
return "\n".join(lines)
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,695 @@
#!/usr/bin/env python3
"""Run a COLMAP SfM preflight report for Real2Sim video keyframes."""
from __future__ import annotations
import argparse
import json
import math
import shutil
import sqlite3
import subprocess
import sys
from pathlib import Path
from typing import Any
from PIL import Image, ImageDraw
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"}
class ColmapCommandError(RuntimeError):
def __init__(self, command: list[str], returncode: int) -> None:
super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}")
self.command = command
self.returncode = returncode
def main() -> int:
parser = argparse.ArgumentParser(description="Generate a COLMAP capture-quality report from selected frames")
parser.add_argument("--frames-dir", type=Path, required=True, help="Directory containing selected image frames")
parser.add_argument("--work-dir", type=Path, required=True, help="COLMAP intermediate directory")
parser.add_argument("--output", type=Path, required=True, help="Output JSON report")
parser.add_argument("--colmap-bin", default="colmap")
parser.add_argument("--matcher", choices=["exhaustive", "sequential"], default="sequential")
parser.add_argument("--sequential-overlap", type=int, default=10)
parser.add_argument("--camera-model", default="SIMPLE_PINHOLE")
parser.add_argument("--max-image-size", type=int, default=0)
parser.add_argument("--sift-max-num-features", type=int, default=8192)
parser.add_argument("--sift-peak-threshold", type=float, default=None)
parser.add_argument("--feature-threads", type=int, default=1)
parser.add_argument("--matcher-threads", type=int, default=1)
parser.add_argument("--guided-matching", action="store_true")
parser.add_argument("--mapper-random-seed", type=int, default=0)
parser.add_argument("--min-registered-ratio", type=float, default=0.70)
parser.add_argument("--min-sparse-points", type=int, default=500)
parser.add_argument("--require-pass", action="store_true")
args = parser.parse_args()
report = run_colmap_report(args)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
if args.require_pass and not report["passed"]:
return 1
return 0
def run_colmap_report(args: argparse.Namespace) -> dict[str, Any]:
frames = collect_frames(args.frames_dir)
if not frames:
raise SystemExit(f"frames directory contains no supported images: {args.frames_dir}")
colmap = resolve_colmap(args.colmap_bin)
work = args.work_dir.resolve()
if work.exists():
shutil.rmtree(work)
work.mkdir(parents=True, exist_ok=True)
database = work / "database.db"
sparse = work / "sparse"
sparse.mkdir(parents=True, exist_ok=True)
text_model = work / "sparse_text"
commands: list[dict[str, Any]] = []
warnings: list[str] = []
try:
feature_command = [
colmap,
"feature_extractor",
"--database_path",
database.as_posix(),
"--image_path",
args.frames_dir.resolve().as_posix(),
"--ImageReader.single_camera",
"1",
"--ImageReader.camera_model",
str(args.camera_model).upper(),
]
feature_gpu_option = colmap_option(colmap, "feature_extractor", ["--FeatureExtraction.use_gpu", "--SiftExtraction.use_gpu"])
if feature_gpu_option is not None:
feature_command.extend([feature_gpu_option, "0"])
feature_threads_option = colmap_option(
colmap,
"feature_extractor",
["--FeatureExtraction.num_threads", "--SiftExtraction.num_threads"],
)
if feature_threads_option is not None and int(args.feature_threads) > 0:
feature_command.extend([feature_threads_option, str(int(args.feature_threads))])
if int(args.max_image_size) > 0:
append_option_if_supported(
colmap,
"feature_extractor",
feature_command,
["--SiftExtraction.max_image_size"],
str(int(args.max_image_size)),
)
append_option_if_supported(
colmap,
"feature_extractor",
feature_command,
["--SiftExtraction.max_num_features"],
str(max(int(args.sift_max_num_features), 1)),
)
if args.sift_peak_threshold is not None:
append_option_if_supported(
colmap,
"feature_extractor",
feature_command,
["--SiftExtraction.peak_threshold"],
str(float(args.sift_peak_threshold)),
)
run_logged(feature_command, commands)
matcher_command = build_matcher_command(
colmap,
args.matcher,
database,
sequential_overlap=args.sequential_overlap,
matcher_threads=int(args.matcher_threads),
guided_matching=bool(args.guided_matching),
)
run_logged(matcher_command, commands)
mapper_command = [
colmap,
"mapper",
"--database_path",
database.as_posix(),
"--image_path",
args.frames_dir.resolve().as_posix(),
"--output_path",
sparse.as_posix(),
]
append_option_if_supported(
colmap,
"mapper",
mapper_command,
["--Mapper.random_seed"],
str(int(args.mapper_random_seed)),
)
run_logged(mapper_command, commands)
except ColmapCommandError as exc:
warnings.append(str(exc))
model_dir, registered_images, sparse_points, model_candidates = select_best_model_dir(
colmap,
sparse,
text_model,
commands,
)
if model_dir is None:
warnings.append("COLMAP mapper produced no sparse model")
registered_ratio = len(registered_images) / max(float(len(frames)), 1.0)
passed = registered_ratio >= args.min_registered_ratio and sparse_points >= args.min_sparse_points
if registered_ratio < args.min_registered_ratio:
warnings.append(
f"registered image ratio {registered_ratio:.3f} < {float(args.min_registered_ratio):.3f}"
)
if sparse_points < args.min_sparse_points:
warnings.append(f"sparse point count {sparse_points} < {int(args.min_sparse_points)}")
database_stats = read_database_stats(database)
geometry = read_best_model_geometry(text_model, work, warnings)
return {
"format": "gs-playground-real2sim-colmap-capture-report-v1",
"frames_dir": args.frames_dir.resolve().as_posix(),
"frame_count": len(frames),
"frame_names": [path.name for path in frames],
"colmap_bin": colmap,
"matcher": args.matcher,
"camera_model": str(args.camera_model).upper(),
"max_image_size": int(args.max_image_size),
"sift_max_num_features": int(args.sift_max_num_features),
"sift_peak_threshold": float(args.sift_peak_threshold) if args.sift_peak_threshold is not None else None,
"feature_threads": int(args.feature_threads),
"matcher_threads": int(args.matcher_threads),
"model_dir": model_dir.as_posix() if model_dir is not None else None,
"model_candidates": model_candidates,
"registered_image_count": len(registered_images),
"registered_ratio": round(float(registered_ratio), 4),
"registered_images": registered_images,
"sparse_point_count": int(sparse_points),
"best_model_text_dir": text_model.as_posix() if model_dir is not None else None,
"camera_models": geometry["camera_models"],
"camera_poses": geometry["camera_poses"],
"camera_path": geometry["camera_path"],
"sparse_bounds": geometry["sparse_bounds"],
"coverage_preview": geometry["coverage_preview"],
"database_stats": database_stats,
"gates": {
"min_registered_ratio": float(args.min_registered_ratio),
"min_sparse_points": int(args.min_sparse_points),
},
"passed": bool(passed),
"warnings": warnings,
"recommendations": capture_recommendations(
passed=bool(passed),
registered_ratio=registered_ratio,
sparse_points=sparse_points,
database_stats=database_stats,
warnings=warnings,
),
"commands": commands,
}
def collect_frames(frames_dir: Path) -> list[Path]:
if not frames_dir.is_dir():
raise SystemExit(f"frames directory not found: {frames_dir}")
return sorted(path for path in frames_dir.iterdir() if is_frame_image(path))
def is_frame_image(path: Path) -> bool:
if not path.is_file() or path.suffix.lower() not in IMAGE_SUFFIXES:
return False
name = path.name.lower()
return "contact_sheet" not in name and "preview" not in name
def resolve_colmap(colmap_bin: str) -> str:
path = shutil.which(colmap_bin)
if path is not None:
return path
candidate = Path(colmap_bin)
if candidate.is_file():
return candidate.resolve().as_posix()
raise SystemExit(f"COLMAP binary not found: {colmap_bin}")
def run_logged(command: list[str], commands: list[dict[str, Any]]) -> None:
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
commands.append(
{
"argv": list(command),
"returncode": completed.returncode,
"output_tail": "\n".join(completed.stdout.splitlines()[-80:]),
}
)
if completed.returncode != 0:
sys.stderr.write(completed.stdout)
raise ColmapCommandError(command, completed.returncode)
def build_matcher_command(
colmap: str,
matcher: str,
database: Path,
*,
sequential_overlap: int,
matcher_threads: int,
guided_matching: bool,
) -> list[str]:
command_name = "sequential_matcher" if matcher == "sequential" else "exhaustive_matcher"
command = [
colmap,
command_name,
"--database_path",
database.as_posix(),
]
matcher_gpu_option = colmap_option(colmap, command_name, ["--FeatureMatching.use_gpu", "--SiftMatching.use_gpu"])
if matcher_gpu_option is not None:
command.extend([matcher_gpu_option, "0"])
matcher_threads_option = colmap_option(
colmap,
command_name,
["--FeatureMatching.num_threads", "--SiftMatching.num_threads"],
)
if matcher_threads_option is not None and matcher_threads > 0:
command.extend([matcher_threads_option, str(matcher_threads)])
if guided_matching:
append_option_if_supported(
colmap,
command_name,
command,
["--FeatureMatching.guided_matching", "--SiftMatching.guided_matching"],
"1",
)
append_cpu_bruteforce_option(colmap, command_name, command)
if command_name == "sequential_matcher":
overlap_option = colmap_option(colmap, command_name, ["--SequentialMatching.overlap"])
if overlap_option is not None:
command.extend([overlap_option, str(max(int(sequential_overlap), 1))])
return command
def colmap_option(colmap: str, command: str, names: list[str]) -> str | None:
completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
help_text = completed.stdout
for name in names:
if name in help_text:
return name
return None
def append_cpu_bruteforce_option(colmap: str, command: str, argv: list[str]) -> None:
option = colmap_option(colmap, command, ["--SiftMatching.cpu_brute_force_matcher"])
if option is not None:
argv.extend([option, "1"])
def append_option_if_supported(
colmap: str,
command: str,
argv: list[str],
option_names: list[str],
value: str,
) -> None:
option = colmap_option(colmap, command, option_names)
if option is not None:
argv.extend([option, value])
def select_best_model_dir(
colmap: str,
sparse_dir: Path,
text_model: Path,
commands: list[dict[str, Any]],
) -> tuple[Path | None, list[str], int, list[dict[str, Any]]]:
candidates = sorted(path for path in sparse_dir.iterdir() if path.is_dir()) if sparse_dir.is_dir() else []
if not candidates:
return None, [], 0, []
candidate_text_root = text_model.parent / f"{text_model.name}_candidates"
if candidate_text_root.exists():
shutil.rmtree(candidate_text_root)
candidate_text_root.mkdir(parents=True, exist_ok=True)
scored: list[tuple[int, int, Path, Path, list[str]]] = []
summaries: list[dict[str, Any]] = []
for candidate in candidates:
candidate_text = candidate_text_root / candidate.name
candidate_text.mkdir(parents=True, exist_ok=True)
run_logged(
[
colmap,
"model_converter",
"--input_path",
candidate.as_posix(),
"--output_path",
candidate_text.as_posix(),
"--output_type",
"TXT",
],
commands,
)
registered = parse_registered_images(candidate_text / "images.txt")
points = count_sparse_points(candidate_text / "points3D.txt")
summaries.append(
{
"model_dir": candidate.as_posix(),
"registered_image_count": len(registered),
"sparse_point_count": points,
}
)
scored.append((len(registered), points, candidate, candidate_text, registered))
registered_count, points, model_dir, candidate_text, registered_images = max(
scored,
key=lambda item: (item[0], item[1]),
)
if text_model.exists():
shutil.rmtree(text_model)
shutil.copytree(candidate_text, text_model)
return model_dir, registered_images, int(points), summaries
def parse_registered_images(path: Path) -> list[str]:
if not path.is_file():
return []
lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()]
data_lines = [line for line in lines if line and not line.startswith("#")]
registered: list[str] = []
for index in range(0, len(data_lines), 2):
parts = data_lines[index].split(maxsplit=9)
if len(parts) >= 10:
registered.append(parts[9])
return registered
def count_sparse_points(path: Path) -> int:
if not path.is_file():
return 0
return sum(
1
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()
if line.strip() and not line.startswith("#")
)
def read_best_model_geometry(text_model: Path, work_dir: Path, warnings: list[str]) -> dict[str, Any]:
camera_models = parse_camera_models(text_model / "cameras.txt")
camera_poses = parse_camera_poses(text_model / "images.txt")
sparse_points, sparse_bounds = parse_sparse_points(text_model / "points3D.txt")
camera_path = summarize_camera_path(camera_poses)
coverage_preview = None
if camera_poses and sparse_points:
try:
coverage_preview = write_coverage_preview(
camera_poses,
sparse_points,
work_dir / "sfm_capture_coverage.png",
).as_posix()
except Exception as exc:
warnings.append(f"failed to write COLMAP coverage preview: {exc}")
return {
"camera_models": camera_models,
"camera_poses": camera_poses,
"camera_path": camera_path,
"sparse_bounds": sparse_bounds,
"coverage_preview": coverage_preview,
}
def parse_camera_models(path: Path) -> list[dict[str, Any]]:
if not path.is_file():
return []
cameras: list[dict[str, Any]] = []
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 5:
continue
cameras.append(
{
"camera_id": int(parts[0]),
"model": parts[1],
"width": int(parts[2]),
"height": int(parts[3]),
"params": [float(value) for value in parts[4:]],
}
)
return cameras
def parse_camera_poses(path: Path) -> list[dict[str, Any]]:
if not path.is_file():
return []
lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()]
data_lines = [line for line in lines if line and not line.startswith("#")]
poses: list[dict[str, Any]] = []
for index in range(0, len(data_lines), 2):
parts = data_lines[index].split(maxsplit=9)
if len(parts) < 10:
continue
qvec = [float(value) for value in parts[1:5]]
tvec = [float(value) for value in parts[5:8]]
position = camera_center_from_qt(qvec, tvec)
poses.append(
{
"image_id": int(parts[0]),
"image_name": parts[9],
"camera_id": int(parts[8]),
"qvec_wxyz": round_list(qvec),
"tvec": round_list(tvec),
"position": round_list(position),
}
)
return sorted(poses, key=lambda item: item["image_name"])
def camera_center_from_qt(qvec: list[float], tvec: list[float]) -> list[float]:
rotation = qvec_to_rotmat(qvec)
return [
-sum(rotation[row][col] * tvec[row] for row in range(3))
for col in range(3)
]
def qvec_to_rotmat(qvec: list[float]) -> list[list[float]]:
w, x, y, z = qvec
return [
[1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w],
[2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w],
[2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y],
]
def parse_sparse_points(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if not path.is_file():
return [], {}
points: list[dict[str, Any]] = []
mins = [math.inf, math.inf, math.inf]
maxs = [-math.inf, -math.inf, -math.inf]
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 7:
continue
xyz = [float(parts[1]), float(parts[2]), float(parts[3])]
color = [int(float(parts[4])), int(float(parts[5])), int(float(parts[6]))]
for axis in range(3):
mins[axis] = min(mins[axis], xyz[axis])
maxs[axis] = max(maxs[axis], xyz[axis])
if len(points) < 20000:
points.append({"xyz": xyz, "rgb": color})
if any(not math.isfinite(value) for value in mins + maxs):
return points, {}
center = [(mins[axis] + maxs[axis]) / 2.0 for axis in range(3)]
extent = [maxs[axis] - mins[axis] for axis in range(3)]
robust_mins = [percentile([point["xyz"][axis] for point in points], 0.02) for axis in range(3)]
robust_maxs = [percentile([point["xyz"][axis] for point in points], 0.98) for axis in range(3)]
return points, {
"bounds_min": round_list(mins),
"bounds_max": round_list(maxs),
"bounds_center": round_list(center),
"bounds_extent": round_list(extent),
"robust_bounds_min": round_list(robust_mins),
"robust_bounds_max": round_list(robust_maxs),
"robust_bounds_extent": round_list([robust_maxs[axis] - robust_mins[axis] for axis in range(3)]),
"preview_point_count": len(points),
}
def summarize_camera_path(camera_poses: list[dict[str, Any]]) -> dict[str, Any]:
if not camera_poses:
return {}
positions = [pose["position"] for pose in camera_poses]
mins = [min(position[axis] for position in positions) for axis in range(3)]
maxs = [max(position[axis] for position in positions) for axis in range(3)]
path_length = sum(
distance(positions[index - 1], positions[index])
for index in range(1, len(positions))
)
return {
"pose_count": len(camera_poses),
"path_length": round(float(path_length), 6),
"start_image": camera_poses[0]["image_name"],
"end_image": camera_poses[-1]["image_name"],
"bounds_min": round_list(mins),
"bounds_max": round_list(maxs),
"bounds_extent": round_list([maxs[axis] - mins[axis] for axis in range(3)]),
}
def write_coverage_preview(camera_poses: list[dict[str, Any]], sparse_points: list[dict[str, Any]], output: Path) -> Path:
all_xyz = [point["xyz"] for point in sparse_points] + [pose["position"] for pose in camera_poses]
extents = [
max(point[axis] for point in all_xyz) - min(point[axis] for point in all_xyz)
for axis in range(3)
]
axes = sorted(range(3), key=lambda axis: extents[axis], reverse=True)[:2]
axis_x, axis_y = axes[0], axes[1]
xs = [point[axis_x] for point in all_xyz]
ys = [point[axis_y] for point in all_xyz]
min_x, max_x = robust_axis_bounds(xs)
min_y, max_y = robust_axis_bounds(ys)
pad_x = max((max_x - min_x) * 0.08, 1e-6)
pad_y = max((max_y - min_y) * 0.08, 1e-6)
min_x -= pad_x
max_x += pad_x
min_y -= pad_y
max_y += pad_y
width, height = 1200, 820
margin = 70
image = Image.new("RGB", (width, height), "white")
draw = ImageDraw.Draw(image)
draw.rectangle((margin, margin, width - margin, height - margin), outline=(210, 210, 210))
def project(point: list[float]) -> tuple[int, int]:
px = margin + (point[axis_x] - min_x) / max(max_x - min_x, 1e-9) * (width - 2 * margin)
py = height - margin - (point[axis_y] - min_y) / max(max_y - min_y, 1e-9) * (height - 2 * margin)
return clamp_int(px, 0, width - 1), clamp_int(py, 0, height - 1)
for point in sparse_points:
x, y = project(point["xyz"])
rgb = point.get("rgb", [150, 150, 150])
color = tuple(max(45, min(210, int(value))) for value in rgb)
draw.point((x, y), fill=color)
camera_xy = [project(pose["position"]) for pose in camera_poses]
if len(camera_xy) >= 2:
draw.line(camera_xy, fill=(28, 98, 201), width=3)
for idx, (x, y) in enumerate(camera_xy):
radius = 5 if idx not in {0, len(camera_xy) - 1} else 8
color = (220, 48, 48) if idx == 0 else (34, 148, 83) if idx == len(camera_xy) - 1 else (28, 98, 201)
draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=color, outline=(255, 255, 255))
draw.text((margin, 22), "COLMAP capture coverage: sparse points + registered camera path", fill=(20, 20, 20))
draw.text((margin, height - 44), f"Axes: {axis_name(axis_x)} / {axis_name(axis_y)} Cameras: {len(camera_poses)} Preview points: {len(sparse_points)}", fill=(70, 70, 70))
if camera_xy:
sx, sy = camera_xy[0]
ex, ey = camera_xy[-1]
draw.text((sx + 10, sy + 10), "start", fill=(140, 30, 30))
draw.text((ex + 10, ey + 10), "end", fill=(20, 105, 55))
output.parent.mkdir(parents=True, exist_ok=True)
image.save(output)
return output
def axis_name(axis: int) -> str:
return ("x", "y", "z")[axis]
def robust_axis_bounds(values: list[float]) -> tuple[float, float]:
if not values:
return 0.0, 1.0
low = percentile(values, 0.02)
high = percentile(values, 0.98)
if high <= low:
return min(values), max(values)
return low, high
def percentile(values: list[float], fraction: float) -> float:
if not values:
return 0.0
ordered = sorted(float(value) for value in values)
index = min(max(int(round((len(ordered) - 1) * fraction)), 0), len(ordered) - 1)
return ordered[index]
def clamp_int(value: float, low: int, high: int) -> int:
return max(low, min(high, int(round(value))))
def distance(a: list[float], b: list[float]) -> float:
return math.sqrt(sum((a[axis] - b[axis]) ** 2 for axis in range(3)))
def round_list(values: list[float], digits: int = 6) -> list[float]:
return [round(float(value), digits) for value in values]
def read_database_stats(database: Path) -> dict[str, Any]:
if not database.is_file():
return {}
try:
connection = sqlite3.connect(database.as_posix())
try:
image_count = table_count(connection, "images")
keypoint_rows = connection.execute("SELECT rows FROM keypoints").fetchall()
match_rows = connection.execute("SELECT rows FROM matches").fetchall()
geometry_rows = connection.execute("SELECT rows FROM two_view_geometries").fetchall()
finally:
connection.close()
except sqlite3.Error:
return {}
keypoint_counts = [int(row[0]) for row in keypoint_rows]
match_counts = [int(row[0]) for row in match_rows if int(row[0]) > 0]
geometry_counts = [int(row[0]) for row in geometry_rows if int(row[0]) > 0]
return {
"image_count": image_count,
"keypoint_total": int(sum(keypoint_counts)),
"keypoint_mean": round(float(sum(keypoint_counts) / max(len(keypoint_counts), 1)), 2),
"matched_pair_count": len(match_counts),
"matched_pair_mean": round(float(sum(match_counts) / max(len(match_counts), 1)), 2) if match_counts else 0.0,
"verified_pair_count": len(geometry_counts),
"verified_pair_mean": round(float(sum(geometry_counts) / max(len(geometry_counts), 1)), 2) if geometry_counts else 0.0,
}
def table_count(connection: sqlite3.Connection, table: str) -> int:
return int(connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
def capture_recommendations(
*,
passed: bool,
registered_ratio: float,
sparse_points: int,
database_stats: dict[str, Any],
warnings: list[str],
) -> list[str]:
if passed:
return ["COLMAP registered enough frames for a pose-initialized reconstruction path."]
recommendations = []
warning_text = "\n".join(warnings).lower()
verified_pairs = int(database_stats.get("verified_pair_count", 0) or 0)
if "no sparse model" in warning_text or "mapper" in warning_text:
recommendations.append("Capture slower video with stronger sideways parallax; avoid pure forward motion or mostly rotational sweeps.")
if verified_pairs < 3:
recommendations.append("Increase overlap between adjacent views and keep textured surfaces in frame so COLMAP can verify image pairs.")
if registered_ratio < 0.7:
recommendations.append("Collect 30-80 sharp frames around the same room volume before attempting commercial-grade reconstruction.")
if sparse_points < 500:
recommendations.append("Avoid motion blur, reflective glass, blank walls, and very close foreground occluders during capture.")
return recommendations
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""Run COLMAP Delaunay meshing from an existing dense workspace."""
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any
class ColmapCommandError(RuntimeError):
def __init__(self, command: list[str], returncode: int) -> None:
super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}")
self.command = command
self.returncode = returncode
def main() -> int:
parser = argparse.ArgumentParser(description="Run COLMAP Delaunay meshing from an existing dense workspace")
parser.add_argument("--dense-workspace", type=Path, required=True, help="COLMAP dense workspace containing images/stereo/sparse")
parser.add_argument("--output-ply", type=Path, required=True, help="Output Delaunay mesh PLY")
parser.add_argument("--report", type=Path, required=True, help="Output Delaunay report JSON")
parser.add_argument("--colmap-bin", default="colmap")
parser.add_argument("--fused-ply", type=Path, help="Existing fused point cloud; defaults to dense-workspace/fused.ply")
parser.add_argument("--ensure-vis", action="store_true", help="Run stereo_fusion into the workspace when fused.ply.vis is missing")
parser.add_argument("--geom-consistency", action="store_true", default=True)
parser.add_argument("--no-geom-consistency", action="store_false", dest="geom_consistency")
parser.add_argument("--stereo-fusion-min-num-pixels", type=int, default=5)
parser.add_argument("--stereo-fusion-max-reproj-error", type=float, default=2.0)
parser.add_argument("--stereo-fusion-max-depth-error", type=float, default=0.01)
parser.add_argument("--max-proj-dist", type=float, default=12.0)
parser.add_argument("--max-depth-dist", type=float, default=0.035)
parser.add_argument("--visibility-sigma", type=float, default=3.0)
parser.add_argument("--quality-regularization", type=float, default=1.0)
parser.add_argument("--max-side-length-factor", type=float, default=20.0)
parser.add_argument("--max-side-length-percentile", type=float, default=95.0)
parser.add_argument("--min-mesh-faces", type=int, default=100000)
parser.add_argument("--require-pass", action="store_true")
args = parser.parse_args()
report = run_delaunay_meshing(args)
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
if args.require_pass and not report.get("passed", False):
return 1
return 0
def run_delaunay_meshing(args: argparse.Namespace) -> dict[str, Any]:
colmap = resolve_colmap(args.colmap_bin)
dense_workspace = args.dense_workspace.resolve()
output_ply = args.output_ply.resolve()
fused_ply = (args.fused_ply or dense_workspace / "fused.ply").resolve()
workspace_fused = dense_workspace / "fused.ply"
workspace_vis = dense_workspace / "fused.ply.vis"
commands: list[dict[str, Any]] = []
warnings: list[str] = []
if not dense_workspace.is_dir():
warnings.append(f"dense workspace is missing: {dense_workspace}")
else:
output_ply.parent.mkdir(parents=True, exist_ok=True)
try:
prepare_workspace_fused(
colmap=colmap,
dense_workspace=dense_workspace,
fused_ply=fused_ply,
workspace_fused=workspace_fused,
workspace_vis=workspace_vis,
args=args,
commands=commands,
)
run_logged(build_delaunay_command(colmap, dense_workspace, output_ply, args), commands)
except ColmapCommandError as exc:
warnings.append(str(exc))
mesh_counts = ply_counts(output_ply) if output_ply.is_file() else {}
mesh_face_count = int(mesh_counts.get("face_count") or 0)
passed = mesh_face_count >= int(args.min_mesh_faces)
if mesh_face_count and not passed:
warnings.append(f"Delaunay mesh face count {mesh_face_count} < {int(args.min_mesh_faces)}")
if not mesh_face_count and not any("COLMAP command failed" in warning for warning in warnings):
warnings.append("COLMAP Delaunay meshing produced no mesh")
return {
"format": "gs-playground-real2sim-colmap-delaunay-report-v1",
"dense_workspace": dense_workspace.as_posix(),
"fused_ply": workspace_fused.as_posix() if workspace_fused.is_file() else None,
"fused_visibility": workspace_vis.as_posix() if workspace_vis.is_file() else None,
"output_ply": output_ply.as_posix() if output_ply.is_file() else None,
"mesh_vertex_count": int(mesh_counts.get("vertex_count") or 0),
"mesh_face_count": mesh_face_count,
"min_mesh_faces": int(args.min_mesh_faces),
"ensure_vis": bool(args.ensure_vis),
"geom_consistency": bool(args.geom_consistency),
"delaunay_options": {
"max_proj_dist": float(args.max_proj_dist),
"max_depth_dist": float(args.max_depth_dist),
"visibility_sigma": float(args.visibility_sigma),
"quality_regularization": float(args.quality_regularization),
"max_side_length_factor": float(args.max_side_length_factor),
"max_side_length_percentile": float(args.max_side_length_percentile),
},
"passed": bool(passed),
"warnings": warnings,
"commands": commands,
}
def prepare_workspace_fused(
*,
colmap: str,
dense_workspace: Path,
fused_ply: Path,
workspace_fused: Path,
workspace_vis: Path,
args: argparse.Namespace,
commands: list[dict[str, Any]],
) -> None:
if workspace_fused.resolve() != fused_ply.resolve() and fused_ply.is_file():
workspace_fused.unlink(missing_ok=True)
try:
workspace_fused.symlink_to(fused_ply)
except OSError:
shutil.copy2(fused_ply, workspace_fused)
if workspace_fused.is_file() and workspace_vis.is_file():
return
if not args.ensure_vis:
raise ColmapCommandError(["missing", workspace_vis.as_posix()], 2)
workspace_fused.unlink(missing_ok=True)
run_logged(build_stereo_fusion_command(colmap, dense_workspace, workspace_fused, args), commands)
def build_stereo_fusion_command(
colmap: str,
dense_workspace: Path,
workspace_fused: Path,
args: argparse.Namespace,
) -> list[str]:
command = [
colmap,
"stereo_fusion",
"--workspace_path",
dense_workspace.as_posix(),
"--workspace_format",
"COLMAP",
"--input_type",
"geometric" if args.geom_consistency else "photometric",
"--output_type",
"PLY",
"--output_path",
workspace_fused.as_posix(),
]
append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.min_num_pixels"], str(max(int(args.stereo_fusion_min_num_pixels), 1)))
append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.max_reproj_error"], str(max(float(args.stereo_fusion_max_reproj_error), 0.1)))
append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.max_depth_error"], str(max(float(args.stereo_fusion_max_depth_error), 0.0001)))
return command
def build_delaunay_command(colmap: str, dense_workspace: Path, output_ply: Path, args: argparse.Namespace) -> list[str]:
command = [
colmap,
"delaunay_mesher",
"--input_path",
dense_workspace.as_posix(),
"--input_type",
"dense",
"--output_path",
output_ply.as_posix(),
]
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_proj_dist"], str(max(float(args.max_proj_dist), 0.1)))
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_depth_dist"], str(max(float(args.max_depth_dist), 0.0001)))
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.visibility_sigma"], str(max(float(args.visibility_sigma), 0.1)))
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.quality_regularization"], str(max(float(args.quality_regularization), 0.0)))
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_side_length_factor"], str(max(float(args.max_side_length_factor), 1.0)))
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_side_length_percentile"], str(min(max(float(args.max_side_length_percentile), 1.0), 100.0)))
return command
def run_logged(command: list[str], commands: list[dict[str, Any]], *, required: bool = True) -> None:
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
commands.append(
{
"argv": list(command),
"returncode": completed.returncode,
"output_tail": "\n".join(completed.stdout.splitlines()[-100:]),
}
)
if completed.returncode != 0 and required:
sys.stderr.write(completed.stdout)
raise ColmapCommandError(command, completed.returncode)
def resolve_colmap(colmap_bin: str) -> str:
path = shutil.which(colmap_bin)
if path is not None:
return path
candidate = Path(colmap_bin)
if candidate.is_file():
return candidate.resolve().as_posix()
raise SystemExit(f"COLMAP binary not found: {colmap_bin}")
def append_option_if_supported(
colmap: str,
command: str,
argv: list[str],
option_names: list[str],
value: str,
) -> None:
option = colmap_option(colmap, command, option_names)
if option is not None:
argv.extend([option, value])
def colmap_option(colmap: str, command: str, names: list[str]) -> str | None:
completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
help_text = completed.stdout
for name in names:
if name in help_text:
return name
return None
def ply_counts(path: Path) -> dict[str, int]:
counts = {"vertex_count": 0, "face_count": 0}
if not path.is_file():
return counts
with path.open("rb") as handle:
for raw in handle:
line = raw.decode("ascii", errors="ignore").strip()
if line.startswith("element vertex "):
counts["vertex_count"] = int(line.split()[-1])
elif line.startswith("element face "):
counts["face_count"] = int(line.split()[-1])
elif line == "end_header":
break
return counts
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,317 @@
#!/usr/bin/env python3
"""Run COLMAP dense MVS from a passing Real2Sim SfM report."""
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any
class ColmapCommandError(RuntimeError):
def __init__(self, command: list[str], returncode: int) -> None:
super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}")
self.command = command
self.returncode = returncode
def main() -> int:
parser = argparse.ArgumentParser(description="Run COLMAP dense MVS from a Real2Sim SfM report")
parser.add_argument("--sfm-report", type=Path, required=True, help="SfM report produced by real2sim_colmap_capture_report.py")
parser.add_argument("--output-dir", type=Path, required=True, help="Dense artifact output directory")
parser.add_argument("--report", type=Path, required=True, help="Output dense reconstruction report JSON")
parser.add_argument("--colmap-bin", default="colmap")
parser.add_argument("--max-image-size", type=int, default=960)
parser.add_argument("--min-fused-points", type=int, default=100000)
parser.add_argument("--geom-consistency", action="store_true", default=True)
parser.add_argument("--no-geom-consistency", action="store_false", dest="geom_consistency")
parser.add_argument("--gpu-index", default="")
parser.add_argument("--patch-match-window-radius", type=int, default=5)
parser.add_argument("--patch-match-window-step", type=int, default=1)
parser.add_argument("--patch-match-num-samples", type=int, default=15)
parser.add_argument("--stereo-fusion-min-num-pixels", type=int, default=5)
parser.add_argument("--stereo-fusion-max-reproj-error", type=float, default=2.0)
parser.add_argument("--stereo-fusion-max-depth-error", type=float, default=0.01)
parser.add_argument("--run-poisson", action="store_true")
parser.add_argument("--poisson-depth", type=int, default=10)
parser.add_argument("--require-pass", action="store_true")
args = parser.parse_args()
report = run_dense_reconstruction(args)
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
if args.require_pass and not report.get("passed", False):
return 1
return 0
def run_dense_reconstruction(args: argparse.Namespace) -> dict[str, Any]:
sfm = json.loads(args.sfm_report.read_text(encoding="utf-8"))
colmap = resolve_colmap(args.colmap_bin)
output_dir = args.output_dir.resolve()
dense_workspace = output_dir / "workspace"
fused_ply = output_dir / "fused.ply"
mesh_ply = output_dir / "mesh_poisson.ply"
commands: list[dict[str, Any]] = []
warnings: list[str] = []
if output_dir.exists():
shutil.rmtree(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
frames_dir = Path(str(sfm.get("frames_dir", ""))).resolve()
model_dir = Path(str(sfm.get("model_dir", ""))).resolve()
if not sfm.get("passed", False):
warnings.append("source SfM report did not pass capture gates")
if not frames_dir.is_dir():
warnings.append(f"SfM frames_dir is missing: {frames_dir}")
if not model_dir.is_dir():
warnings.append(f"SfM model_dir is missing: {model_dir}")
if not warnings:
try:
run_logged(build_image_undistorter_command(colmap, frames_dir, model_dir, dense_workspace, args), commands)
run_logged(build_patch_match_command(colmap, dense_workspace, args), commands)
run_logged(build_stereo_fusion_command(colmap, dense_workspace, fused_ply, args), commands)
if args.run_poisson and fused_ply.is_file():
run_logged(build_poisson_command(colmap, fused_ply, mesh_ply, args), commands, required=False)
except ColmapCommandError as exc:
warnings.append(str(exc))
fused_counts = ply_counts(fused_ply) if fused_ply.is_file() else {}
mesh_counts = ply_counts(mesh_ply) if mesh_ply.is_file() else {}
fused_point_count = int(fused_counts.get("vertex_count") or 0)
passed = fused_point_count >= int(args.min_fused_points)
if fused_point_count and not passed:
warnings.append(f"fused point count {fused_point_count} < {int(args.min_fused_points)}")
if not fused_point_count and not any("COLMAP command failed" in warning for warning in warnings):
warnings.append("COLMAP dense reconstruction produced no fused point cloud")
return {
"format": "gs-playground-real2sim-colmap-dense-report-v1",
"sfm_report": args.sfm_report.resolve().as_posix(),
"frames_dir": frames_dir.as_posix(),
"model_dir": model_dir.as_posix(),
"output_dir": output_dir.as_posix(),
"dense_workspace": dense_workspace.as_posix(),
"fused_ply": fused_ply.as_posix() if fused_ply.is_file() else None,
"mesh_ply": mesh_ply.as_posix() if mesh_ply.is_file() else None,
"max_image_size": int(args.max_image_size),
"geom_consistency": bool(args.geom_consistency),
"gpu_index": str(args.gpu_index),
"fused_point_count": fused_point_count,
"mesh_vertex_count": int(mesh_counts.get("vertex_count") or 0),
"mesh_face_count": int(mesh_counts.get("face_count") or 0),
"min_fused_points": int(args.min_fused_points),
"passed": bool(passed),
"warnings": warnings,
"commands": commands,
}
def build_image_undistorter_command(
colmap: str,
frames_dir: Path,
model_dir: Path,
dense_workspace: Path,
args: argparse.Namespace,
) -> list[str]:
command = [
colmap,
"image_undistorter",
"--image_path",
frames_dir.as_posix(),
"--input_path",
model_dir.as_posix(),
"--output_path",
dense_workspace.as_posix(),
"--output_type",
"COLMAP",
]
if int(args.max_image_size) > 0:
append_option_if_supported(
colmap,
"image_undistorter",
command,
["--max_image_size", "--ImageUndistorter.max_image_size"],
str(int(args.max_image_size)),
)
return command
def build_patch_match_command(colmap: str, dense_workspace: Path, args: argparse.Namespace) -> list[str]:
command = [
colmap,
"patch_match_stereo",
"--workspace_path",
dense_workspace.as_posix(),
"--workspace_format",
"COLMAP",
]
append_option_if_supported(
colmap,
"patch_match_stereo",
command,
["--PatchMatchStereo.geom_consistency"],
"true" if args.geom_consistency else "false",
)
if str(args.gpu_index).strip():
append_option_if_supported(
colmap,
"patch_match_stereo",
command,
["--PatchMatchStereo.gpu_index"],
str(args.gpu_index).strip(),
)
append_option_if_supported(
colmap,
"patch_match_stereo",
command,
["--PatchMatchStereo.window_radius"],
str(max(int(args.patch_match_window_radius), 1)),
)
append_option_if_supported(
colmap,
"patch_match_stereo",
command,
["--PatchMatchStereo.window_step"],
str(max(int(args.patch_match_window_step), 1)),
)
append_option_if_supported(
colmap,
"patch_match_stereo",
command,
["--PatchMatchStereo.num_samples"],
str(max(int(args.patch_match_num_samples), 1)),
)
return command
def build_stereo_fusion_command(
colmap: str,
dense_workspace: Path,
fused_ply: Path,
args: argparse.Namespace,
) -> list[str]:
command = [
colmap,
"stereo_fusion",
"--workspace_path",
dense_workspace.as_posix(),
"--workspace_format",
"COLMAP",
"--input_type",
"geometric" if args.geom_consistency else "photometric",
"--output_path",
fused_ply.as_posix(),
]
append_option_if_supported(
colmap,
"stereo_fusion",
command,
["--StereoFusion.min_num_pixels"],
str(max(int(args.stereo_fusion_min_num_pixels), 1)),
)
append_option_if_supported(
colmap,
"stereo_fusion",
command,
["--StereoFusion.max_reproj_error"],
str(max(float(args.stereo_fusion_max_reproj_error), 0.1)),
)
append_option_if_supported(
colmap,
"stereo_fusion",
command,
["--StereoFusion.max_depth_error"],
str(max(float(args.stereo_fusion_max_depth_error), 0.0001)),
)
return command
def build_poisson_command(colmap: str, fused_ply: Path, mesh_ply: Path, args: argparse.Namespace) -> list[str]:
command = [
colmap,
"poisson_mesher",
"--input_path",
fused_ply.as_posix(),
"--output_path",
mesh_ply.as_posix(),
]
append_option_if_supported(
colmap,
"poisson_mesher",
command,
["--PoissonMeshing.depth"],
str(max(int(args.poisson_depth), 5)),
)
return command
def run_logged(command: list[str], commands: list[dict[str, Any]], *, required: bool = True) -> None:
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
commands.append(
{
"argv": list(command),
"returncode": completed.returncode,
"output_tail": "\n".join(completed.stdout.splitlines()[-100:]),
}
)
if completed.returncode != 0 and required:
sys.stderr.write(completed.stdout)
raise ColmapCommandError(command, completed.returncode)
def resolve_colmap(colmap_bin: str) -> str:
path = shutil.which(colmap_bin)
if path is not None:
return path
candidate = Path(colmap_bin)
if candidate.is_file():
return candidate.resolve().as_posix()
raise SystemExit(f"COLMAP binary not found: {colmap_bin}")
def colmap_option(colmap: str, command: str, names: list[str]) -> str | None:
completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
help_text = completed.stdout
for name in names:
if name in help_text:
return name
return None
def append_option_if_supported(
colmap: str,
command: str,
argv: list[str],
option_names: list[str],
value: str,
) -> None:
option = colmap_option(colmap, command, option_names)
if option is not None:
argv.extend([option, value])
def ply_counts(path: Path) -> dict[str, int]:
counts = {"vertex_count": 0, "face_count": 0}
if not path.is_file():
return counts
with path.open("rb") as handle:
for raw in handle:
line = raw.decode("ascii", errors="ignore").strip()
if line.startswith("element vertex "):
counts["vertex_count"] = int(line.split()[-1])
elif line.startswith("element face "):
counts["face_count"] = int(line.split()[-1])
elif line == "end_header":
break
return counts
if __name__ == "__main__":
raise SystemExit(main())

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,68 @@
#!/usr/bin/env bash
# Download SAM-3D Objects checkpoints into the layout expected by Real2Sim.
#
# Official paper-equivalent source:
# REAL2SIM_SAM3D_CHECKPOINT_REPO=facebook/sam-3d-objects
#
# Public mirror fallback used only when the official gated repo is unavailable:
# REAL2SIM_SAM3D_CHECKPOINT_REPO=licesma/sam-3d-objects-weights
set -euo pipefail
if [[ -z "${REAL2SIM_SAM3D_ROOT:-}" ]]; then
echo "REAL2SIM_SAM3D_ROOT is required. Source scripts/real2sim_wuying_paper_env.sh first." >&2
exit 2
fi
repo="${REAL2SIM_SAM3D_CHECKPOINT_REPO:-facebook/sam-3d-objects}"
max_workers="${REAL2SIM_SAM3D_DOWNLOAD_WORKERS:-1}"
dest="${REAL2SIM_SAM3D_CHECKPOINT_DIR:-${REAL2SIM_SAM3D_ROOT}/checkpoints/hf}"
hf_bin="${REAL2SIM_HF_BIN:-hf}"
force="${REAL2SIM_SAM3D_CHECKPOINT_FORCE:-0}"
if [[ -e "${dest}/pipeline.yaml" && "${force}" != "1" ]]; then
echo "SAM-3D checkpoints already exist at ${dest}. Set REAL2SIM_SAM3D_CHECKPOINT_FORCE=1 to replace." >&2
exit 0
fi
mkdir -p "${REAL2SIM_SAM3D_ROOT}/checkpoints"
download_dir="$(mktemp -d "${REAL2SIM_SAM3D_ROOT}/checkpoints/.sam3d-download.XXXXXX")"
cleanup() {
rm -rf "${download_dir}"
}
trap cleanup EXIT
"${hf_bin}" download \
"${repo}" \
--repo-type model \
--local-dir "${download_dir}" \
--max-workers "${max_workers}"
source_dir=""
if [[ -f "${download_dir}/checkpoints/pipeline.yaml" ]]; then
source_dir="${download_dir}/checkpoints"
elif [[ -f "${download_dir}/hf/pipeline.yaml" ]]; then
source_dir="${download_dir}/hf"
else
echo "Downloaded ${repo}, but no checkpoints/pipeline.yaml or hf/pipeline.yaml was found." >&2
find "${download_dir}" -maxdepth 3 -type f | sort >&2
exit 1
fi
tmp_dest="${dest}.tmp.$$"
rm -rf "${tmp_dest}"
mkdir -p "$(dirname "${dest}")"
cp -a "${source_dir}" "${tmp_dest}"
if [[ -e "${dest}" || -L "${dest}" ]]; then
rm -rf "${dest}"
fi
mv "${tmp_dest}" "${dest}"
cat > "${dest}.SOURCE.txt" <<EOF
repo=${repo}
downloaded_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
destination=${dest}
official_repo=facebook/sam-3d-objects
EOF
echo "SAM-3D checkpoints are ready at ${dest}"

View File

@ -0,0 +1,281 @@
#!/usr/bin/env python3
"""Export a COLMAP pose-initialized training dataset from a capture report."""
from __future__ import annotations
import argparse
import json
import math
import shutil
from pathlib import Path
from typing import Any
COLMAP_TEXT_FILES = ("cameras.txt", "images.txt", "points3D.txt")
def main() -> int:
parser = argparse.ArgumentParser(description="Export registered COLMAP frames as a pose-init 3DGS training dataset")
parser.add_argument("--sfm-report", type=Path, required=True, help="COLMAP capture report JSON")
parser.add_argument("--output-dir", type=Path, required=True, help="Output pose-init dataset directory")
parser.add_argument("--allow-failed-sfm", action="store_true", help="Export even when sfm_report.passed is false")
args = parser.parse_args()
summary = export_pose_init_dataset(
sfm_report=args.sfm_report,
output_dir=args.output_dir,
allow_failed_sfm=bool(args.allow_failed_sfm),
)
print(json.dumps(summary, indent=2, ensure_ascii=False))
return 0
def export_pose_init_dataset(*, sfm_report: Path, output_dir: Path, allow_failed_sfm: bool = False) -> dict[str, Any]:
report = read_json(sfm_report)
if not report.get("passed") and not allow_failed_sfm:
raise SystemExit("SFM report did not pass; rerun with --allow-failed-sfm to export anyway")
frames_dir = Path(str(report.get("frames_dir", "")))
if not frames_dir.is_dir():
raise SystemExit(f"SFM report frames_dir does not exist: {frames_dir}")
text_model = Path(str(report.get("best_model_text_dir", "")))
if not text_model.is_dir():
raise SystemExit(f"SFM report best_model_text_dir does not exist: {text_model}")
camera_models = {int(item["camera_id"]): item for item in report.get("camera_models", [])}
camera_poses = list(report.get("camera_poses", []))
if not camera_models:
raise SystemExit("SFM report contains no camera models")
if not camera_poses:
raise SystemExit("SFM report contains no registered camera poses")
if output_dir.exists():
shutil.rmtree(output_dir)
image_dir = output_dir / "images"
colmap_text_dir = output_dir / "colmap_text"
image_dir.mkdir(parents=True, exist_ok=True)
colmap_text_dir.mkdir(parents=True, exist_ok=True)
frames: list[dict[str, Any]] = []
copied_images = 0
for pose in sorted(camera_poses, key=lambda item: str(item.get("image_name", ""))):
image_name = str(pose.get("image_name", ""))
source_image = frames_dir / image_name
if not source_image.is_file():
raise SystemExit(f"registered COLMAP frame is missing: {source_image}")
dest_image = image_dir / source_image.name
shutil.copy2(source_image, dest_image)
copied_images += 1
camera_id = int(pose["camera_id"])
camera = camera_models.get(camera_id)
if camera is None:
raise SystemExit(f"pose references unknown camera_id {camera_id}: {image_name}")
intrinsics = camera_intrinsics(camera)
colmap_c2w = colmap_camera_to_world(pose["qvec_wxyz"], pose["tvec"])
nerfstudio_c2w = colmap_c2w_to_nerfstudio(colmap_c2w)
frames.append(
{
"file_path": f"images/{dest_image.name}",
"transform_matrix": round_matrix(nerfstudio_c2w),
"colmap_transform_matrix": round_matrix(colmap_c2w),
"fl_x": intrinsics["fl_x"],
"fl_y": intrinsics["fl_y"],
"cx": intrinsics["cx"],
"cy": intrinsics["cy"],
"w": intrinsics["w"],
"h": intrinsics["h"],
"camera_model": intrinsics["camera_model"],
"real2sim_pose": {
"image_id": pose.get("image_id"),
"camera_id": camera_id,
"qvec_wxyz": pose.get("qvec_wxyz"),
"tvec": pose.get("tvec"),
"position_colmap_world": pose.get("position"),
},
}
)
sparse_ply = write_sparse_points_ply(text_model / "points3D.txt", output_dir / "colmap_sparse_points.ply")
copied_colmap_files = copy_colmap_text_model(text_model, colmap_text_dir)
transforms = {
"format": "gs-playground-real2sim-pose-init-transforms-v1",
"camera_convention": "nerfstudio/OpenGL camera-to-world; raw COLMAP camera-to-world is preserved per frame",
"source_sfm_report": sfm_report.resolve().as_posix(),
"frame_count": len(frames),
"registered_image_count": int(report.get("registered_image_count", len(frames))),
"sparse_point_count": int(report.get("sparse_point_count", 0) or 0),
"camera_path": report.get("camera_path", {}),
"sparse_bounds": report.get("sparse_bounds", {}),
"database_stats": report.get("database_stats", {}),
"frames": frames,
}
(output_dir / "transforms.json").write_text(json.dumps(transforms, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
write_readme(output_dir, transforms, sparse_ply=sparse_ply)
summary = {
"format": "gs-playground-real2sim-pose-init-dataset-v1",
"root": output_dir.resolve().as_posix(),
"transforms": "transforms.json",
"images": "images",
"image_count": copied_images,
"colmap_text": "colmap_text",
"colmap_text_files": copied_colmap_files,
"sparse_points_ply": sparse_ply.name if sparse_ply is not None else None,
"sparse_point_count": int(report.get("sparse_point_count", 0) or 0),
"registered_ratio": report.get("registered_ratio"),
"ready_for_training": bool(report.get("passed") and copied_images > 0 and sparse_ply is not None),
}
(output_dir / "pose_init_dataset.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
return summary
def read_json(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
if not isinstance(payload, dict):
raise SystemExit(f"expected JSON object: {path}")
return payload
def camera_intrinsics(camera: dict[str, Any]) -> dict[str, Any]:
model = str(camera.get("model", "")).upper()
width = int(camera["width"])
height = int(camera["height"])
params = [float(value) for value in camera.get("params", [])]
if model == "SIMPLE_PINHOLE":
f, cx, cy = params[:3]
fx = fy = f
elif model == "PINHOLE":
fx, fy, cx, cy = params[:4]
elif model in {"SIMPLE_RADIAL", "RADIAL"}:
f, cx, cy = params[:3]
fx = fy = f
elif model in {"OPENCV", "OPENCV_FISHEYE", "FULL_OPENCV"}:
fx, fy, cx, cy = params[:4]
else:
raise SystemExit(f"unsupported COLMAP camera model for pose-init export: {model}")
return {
"camera_model": model,
"w": width,
"h": height,
"fl_x": round(float(fx), 6),
"fl_y": round(float(fy), 6),
"cx": round(float(cx), 6),
"cy": round(float(cy), 6),
}
def colmap_camera_to_world(qvec: list[float], tvec: list[float]) -> list[list[float]]:
rotation = qvec_to_rotmat([float(value) for value in qvec])
t = [float(value) for value in tvec]
rt = transpose(rotation)
center = [-sum(rt[row][col] * t[col] for col in range(3)) for row in range(3)]
return [
[rt[0][0], rt[0][1], rt[0][2], center[0]],
[rt[1][0], rt[1][1], rt[1][2], center[1]],
[rt[2][0], rt[2][1], rt[2][2], center[2]],
[0.0, 0.0, 0.0, 1.0],
]
def colmap_c2w_to_nerfstudio(matrix: list[list[float]]) -> list[list[float]]:
converted = [list(row) for row in matrix]
for row in range(3):
converted[row][1] *= -1.0
converted[row][2] *= -1.0
return converted
def qvec_to_rotmat(qvec: list[float]) -> list[list[float]]:
w, x, y, z = qvec
return [
[1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w],
[2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w],
[2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y],
]
def transpose(matrix: list[list[float]]) -> list[list[float]]:
return [[matrix[col][row] for col in range(3)] for row in range(3)]
def round_matrix(matrix: list[list[float]], digits: int = 8) -> list[list[float]]:
return [[round(float(value), digits) for value in row] for row in matrix]
def write_sparse_points_ply(points_txt: Path, output: Path) -> Path | None:
points: list[tuple[float, float, float, int, int, int]] = []
if not points_txt.is_file():
return None
for line in points_txt.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 7:
continue
x, y, z = (float(parts[index]) for index in range(1, 4))
r, g, b = (clamp_color(float(parts[index])) for index in range(4, 7))
points.append((x, y, z, r, g, b))
if not points:
return None
with output.open("w", encoding="ascii") as handle:
handle.write("ply\n")
handle.write("format ascii 1.0\n")
handle.write(f"element vertex {len(points)}\n")
handle.write("property float x\n")
handle.write("property float y\n")
handle.write("property float z\n")
handle.write("property uchar red\n")
handle.write("property uchar green\n")
handle.write("property uchar blue\n")
handle.write("end_header\n")
for x, y, z, r, g, b in points:
handle.write(f"{x:.8f} {y:.8f} {z:.8f} {r} {g} {b}\n")
return output
def clamp_color(value: float) -> int:
return max(0, min(255, int(round(value))))
def copy_colmap_text_model(text_model: Path, output: Path) -> list[str]:
copied: list[str] = []
for name in COLMAP_TEXT_FILES:
source = text_model / name
if source.is_file():
shutil.copy2(source, output / name)
copied.append(name)
return copied
def write_readme(output_dir: Path, transforms: dict[str, Any], *, sparse_ply: Path | None) -> None:
lines = [
"# Real2Sim Pose-Init Dataset",
"",
"This directory is generated from a COLMAP capture report that passed the Real2Sim capture gate.",
"It is intended as the bridge from fast AnySplat preview reconstruction to a pose-initialized 3DGS training path.",
"",
"Contents:",
"",
"- `images/`: registered video frames copied from the SfM frame set.",
"- `transforms.json`: Nerfstudio/OpenGL-style camera-to-world transforms plus raw COLMAP pose metadata.",
"- `colmap_text/`: the selected COLMAP text model (`cameras.txt`, `images.txt`, `points3D.txt`).",
"- `colmap_sparse_points.ply`: colored sparse point cloud exported from `points3D.txt`.",
"",
"Summary:",
"",
f"- frames: {transforms.get('frame_count')}",
f"- registered images: {transforms.get('registered_image_count')}",
f"- sparse points: {transforms.get('sparse_point_count')}",
f"- sparse PLY: {sparse_ply.name if sparse_ply is not None else 'missing'}",
"",
"A downstream trainer should initialize from these poses and sparse points instead of random camera/point initialization.",
"",
]
(output_dir / "README.md").write_text("\n".join(lines), encoding="utf-8")
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,115 @@
# Real2Sim External Model Adapters
These scripts connect the GS-Playground Real2Sim stage contract to public model
implementations. They are optional: the built-in backend still runs without any
of these dependencies.
## Quality Upgrade Path
1. `groundingdino_hf_detect.py`
- Purpose: replace heuristic connected components with open-vocabulary boxes.
- Source: official GroundingDINO / Grounded-SAM-2 Hugging Face path.
- Main dependency: `transformers`, `Pillow`, `torch`.
- Default model: `IDEA-Research/grounding-dino-tiny`.
2. `sam2_segment.py`
- Purpose: replace coarse masks with SAM2 box-prompted masks.
- Source: official `facebookresearch/sam2`.
- Required env:
- `REAL2SIM_SAM2_CHECKPOINT=/path/to/sam2.1_hiera_large.pt`
- `REAL2SIM_SAM2_MODEL_CFG=configs/sam2.1/sam2.1_hiera_l.yaml`
3. `instance_refine.py`
- Purpose: match the paper's visual-only instance de-duplication stage after
SAM/SAM2.
- Method: composite detection/segmentation confidence, mask IoU, mask
inclusion, and boundary overlap.
4. `lama_inpaint.py`
- Purpose: replace median-fill background removal with sequential LaMa
inpainting.
- Runtime used here: `simple-lama-inpainting`, a lightweight LaMa wrapper
listed by the official LaMa repository.
- Method: one-object-at-a-time mask expansion and sequential background
update. Re-detection of newly exposed regions must be supplied by the
configured detection/segmentation loop for fully strict paper parity.
The strict config uses `REAL2SIM_GROUNDING_DINO_COMMAND` and
`REAL2SIM_SAM2_COMMAND` for this nested loop.
5. `depth_anything_depth.py`
- Purpose: replace flat pseudo-depth detail with a monocular Depth Anything
V2 depth prior before building the fallback 3DGS assets.
- Main dependency: `transformers`, `Pillow`, `torch`.
- Default model: `depth-anything/Depth-Anything-V2-Small-hf`.
6. `anysplat_scene.py`
- Purpose: replace pseudo-depth scene Gaussians with AnySplat scene Gaussians.
- Source: official AnySplat repo and Hugging Face model `lhjiang/anysplat`.
- Output: official `export_ply` scene Gaussians, decoder depth, camera
metadata, and a depth-derived scene mesh.
- Required env:
- `REAL2SIM_ANYSPLAT_ROOT=/path/to/AnySplat`
- Best input: multiple nearby views or a short video, not a single repeated
frame.
7. `sam3d_objects_reconstruct.py`
- Purpose: replace object box meshes and object pseudo-Gaussians with SAM 3D
Objects outputs.
- Source: official Meta `facebookresearch/sam-3d-objects`.
- Output: object Gaussian PLY from the SAM-3D `save_ply` output, an
exportable object mesh, and pose/scale metadata. The script fails if no
mesh is returned unless `--allow-fallback-mesh` is passed for debugging.
- Required env:
- `REAL2SIM_SAM3D_ROOT=/path/to/sam-3d-objects`
- `REAL2SIM_SAM3D_CONFIG=/path/to/checkpoints/hf/pipeline.yaml`
- Checkpoints are gated on Hugging Face and require accepted access. The
upstream setup currently recommends an NVIDIA GPU with at least 32 GB VRAM.
- Use `--cameras {cameras_json} --match-camera-preprocess` when the scene
was reconstructed by AnySplat, so SAM-3D receives the same
aspect-preserving center crop and mask geometry as the scene camera.
- Use `--canonicalize-gaussian-to-mesh` to rotate the object Gaussian PLY
into the SAM-3D mesh canonical frame using only geometric extent axes. It
updates positions, normals, and splat quaternions without source-image
photometric scoring.
- Use `--remove-video-overlay` only for screenshots/video thumbnails with a
central play button; it inpaints detected bright, low-saturation UI pixels
near the image center before SAM-3D inference.
8. `depth_mask_pose_align.py`
- Purpose: align SAM-3D objects back into the AnySplat scene using rendered
object Gaussian PLY depth/mask, scene depth, and camera intrinsics.
Mesh rendering is used only as a fallback when the object Gaussian asset is
unavailable or invalid.
- Output: an updated `{aligned_assets_json}` manifest with refined
`pose.position`, `pose.quat_wxyz`, and `pose.scale` values plus alignment
metadata including the selected `render_source`, candidate axis search
label, mask IoU, centroid error, depth error, and quality summary.
If the object Gaussian directory contains SAM-3D metadata showing
`gaussian_canonicalization.status=ok`, the aligner keeps the canonical
rotation and uses only a mask/depth position-scale seed instead of
source-view axis search.
9. Speedy-Splat pruning
- Purpose: match the paper's 3DGS pruning stage.
- Configure `pruning.mode=speedy_splat` and set
`pruning.external_command` or `REAL2SIM_SPEEDY_SPLAT_COMMAND`.
- The pipeline fails if the command is missing in Speedy-Splat mode.
- The adapter keeps Speedy-Splat's official score computation, but applies
an exact-count tie-safe prune mask so all-zero or heavily tied object
scores cannot delete every Gaussian in a canonical local object asset.
## Example
```bash
python -m real2sim run \
--backend hybrid \
--config configs/real2sim.external.example.json \
--input media/teaser.png \
--output /tmp/gs_real2sim_hybrid \
--prompt "robot. person. table. chair. object."
```
Use `--backend external` only after AnySplat and SAM 3D Objects are installed,
their checkpoints are available, and the machine meets the SAM-3D memory
requirements. Use `python -m real2sim run ... --dry-run --backend hybrid` to
check which stage commands are configured before launching a long model run.

View File

@ -0,0 +1,223 @@
#!/usr/bin/env python3
"""AnySplat scene reconstruction adapter for Real2Sim.
This adapter uses the official AnySplat model/API and exports a real 3DGS PLY
through AnySplat's `export_ply` helper. Video inputs can provide real nearby
frames through `--frames-dir`; single-image inputs are kept as a single view
instead of pretending that duplicated frames are independent observations.
"""
from __future__ import annotations
import argparse
import os
import sys
from pathlib import Path
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.contracts import write_json
from real2sim.image_io import write_gray
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"}
def main() -> int:
parser = argparse.ArgumentParser(description="Run AnySplat scene reconstruction for Real2Sim")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--ply", type=Path, required=True)
parser.add_argument("--mesh", type=Path, required=True)
parser.add_argument("--cameras", type=Path, required=True)
parser.add_argument("--depth", type=Path, required=True)
parser.add_argument("--source-preview", type=Path, default=None, help="Optional source-camera RGB render preview PNG")
parser.add_argument("--frames-dir", type=Path, default=None, help="Optional extracted video frames directory")
parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT"))
parser.add_argument("--model-id", default=os.environ.get("REAL2SIM_ANYSPLAT_MODEL_ID", "lhjiang/anysplat"))
parser.add_argument("--max-views", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MAX_VIEWS", "8")))
parser.add_argument("--mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MESH_GRID", "48")))
parser.add_argument("--shift-and-scale", action="store_true", help="Normalize exported Gaussian positions with AnySplat exporter")
args = parser.parse_args()
if args.anysplat_root is None:
raise SystemExit("AnySplat root is required via --anysplat-root or REAL2SIM_ANYSPLAT_ROOT")
sys.path.insert(0, args.anysplat_root.as_posix())
import torch
from src.model.model.anysplat import AnySplat
from src.model.ply_export import export_ply
from src.utils.image import process_image
from real2sim.builtin import write_depth_mesh_obj
args.output_dir.mkdir(parents=True, exist_ok=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = AnySplat.from_pretrained(args.model_id).to(device)
model.eval()
for param in model.parameters():
param.requires_grad = False
image_paths = collect_image_paths(args.image, args.frames_dir, max_views=args.max_views)
images = torch.stack([process_image(path.as_posix()) for path in image_paths], dim=0).unsqueeze(0).to(device)
b, views, _, height, width = images.shape
with torch.inference_mode():
gaussians, pose = model.inference((images + 1) * 0.5)
export_ply(
gaussians.means[0],
gaussians.scales[0],
gaussians.rotations[0],
gaussians.harmonics[0],
gaussians.opacities[0],
args.ply,
shift_and_scale=args.shift_and_scale,
save_sh_dc_only=True,
)
extrinsic = pose["extrinsic"]
intrinsic = pose["intrinsic"]
near = torch.ones(b, views, device=device) * 0.1
far = torch.ones(b, views, device=device) * 100.0
with torch.inference_mode():
rendered = model.decoder.forward(gaussians, extrinsic, intrinsic.float(), near, far, (height, width), "depth")
preview_metrics = None
if args.source_preview is not None:
source_rgb_tensor = ((images[0, 0].detach().cpu() + 1.0) * 0.5).clamp(0.0, 1.0)
rendered_rgb_tensor = rendered.color[0, 0].detach().cpu().clamp(0.0, 1.0)
preview_metrics = save_source_preview(args.source_preview, rendered_rgb_tensor, source_rgb_tensor)
depth = extract_first_depth_map(rendered.depth)
normalized_depth = normalize_depth(depth)
write_gray(args.depth, normalized_depth)
intrinsics = denormalize_intrinsics(intrinsic[0, 0].detach().cpu().numpy(), width, height)
scene_rgb = (((images[0, 0].detach().cpu().permute(1, 2, 0).numpy()) + 1.0) * 0.5).clip(0.0, 1.0)
write_depth_mesh_obj(args.mesh, scene_rgb, normalized_depth, intrinsics, grid=max(8, int(args.mesh_grid)))
write_json(
args.cameras,
{
"backend": "anysplat",
"model_id": args.model_id,
"image_paths": [path.as_posix() for path in image_paths],
"image_shape": [height, width],
"intrinsics": intrinsics,
"source_preview": args.source_preview.as_posix() if args.source_preview is not None else None,
"source_preview_metrics": preview_metrics,
"pred_context_pose": to_jsonable(pose),
},
)
write_json(
args.output_dir / "anysplat_metadata.json",
{
"backend": "anysplat",
"model_id": args.model_id,
"num_views": len(image_paths),
"max_views": args.max_views,
"mesh_grid": max(8, int(args.mesh_grid)),
"shift_and_scale": args.shift_and_scale,
"source_preview": args.source_preview.as_posix() if args.source_preview is not None else None,
"source_preview_metrics": preview_metrics,
},
)
return 0
def save_source_preview(output: Path, rendered_rgb, source_rgb) -> dict[str, float]:
from PIL import Image
rendered = tensor_rgb_to_numpy(rendered_rgb)
source = tensor_rgb_to_numpy(source_rgb)
mae = float(np.mean(np.abs(rendered.astype(np.float32) / 255.0 - source.astype(np.float32) / 255.0)))
mse = float(np.mean((rendered.astype(np.float32) / 255.0 - source.astype(np.float32) / 255.0) ** 2))
psnr = float(99.0 if mse <= 1e-12 else -10.0 * np.log10(mse))
output.parent.mkdir(parents=True, exist_ok=True)
Image.fromarray(rendered).save(output)
return {"mae": mae, "mse": mse, "psnr": psnr}
def tensor_rgb_to_numpy(rgb) -> np.ndarray:
if hasattr(rgb, "detach"):
rgb = rgb.detach().cpu().numpy()
arr = np.asarray(rgb, dtype=np.float32)
if arr.ndim == 3 and arr.shape[0] == 3:
arr = np.transpose(arr, (1, 2, 0))
if arr.ndim != 3 or arr.shape[-1] != 3:
raise SystemExit(f"RGB preview tensor must have shape 3xHxW or HxWx3, got {arr.shape}")
return (arr.clip(0.0, 1.0) * 255.0).round().astype(np.uint8)
def collect_image_paths(image: Path, frames_dir: Path | None, *, max_views: int) -> list[Path]:
if frames_dir is not None and str(frames_dir) not in {"", ".", "None", "__none__"} and frames_dir.is_dir():
frames = sorted(path for path in frames_dir.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES)
if frames:
return sample_evenly(frames, max_views=max(1, max_views))
return [image]
def sample_evenly(paths: list[Path], *, max_views: int) -> list[Path]:
if len(paths) <= max_views:
return paths
if max_views == 1:
return [paths[0]]
indices = np.linspace(0, len(paths) - 1, num=max_views).round().astype(int)
return [paths[int(idx)] for idx in sorted(set(indices))]
def normalize_depth(depth: np.ndarray) -> np.ndarray:
finite = np.asarray(depth, dtype=np.float32)
valid = np.isfinite(finite)
if not valid.any():
return np.zeros_like(finite, dtype=np.float32)
lo = float(np.quantile(finite[valid], 0.02))
hi = float(np.quantile(finite[valid], 0.98))
if hi <= lo:
return np.zeros_like(finite, dtype=np.float32)
return np.clip((finite - lo) / (hi - lo), 0.0, 1.0).astype(np.float32)
def extract_first_depth_map(depth) -> np.ndarray:
if hasattr(depth, "detach"):
depth = depth.detach().cpu().numpy()
arr = np.asarray(depth, dtype=np.float32)
if arr.ndim == 5 and arr.shape[-1] == 1:
arr = arr[..., 0]
if arr.ndim == 4:
arr = arr[0, 0]
elif arr.ndim == 3:
arr = arr[0]
elif arr.ndim != 2:
raise SystemExit(f"AnySplat decoder depth must be 2D, 3D, or 4D, got shape {arr.shape}")
if arr.ndim != 2:
raise SystemExit(f"AnySplat decoder depth did not reduce to a 2D map, got shape {arr.shape}")
return arr.astype(np.float32, copy=False)
def denormalize_intrinsics(intrinsic: np.ndarray, width: int, height: int) -> dict[str, float | int]:
intr = np.asarray(intrinsic, dtype=np.float32)
return {
"width": int(width),
"height": int(height),
"fx": float(intr[0, 0] * width),
"fy": float(intr[1, 1] * height),
"cx": float(intr[0, 2] * width),
"cy": float(intr[1, 2] * height),
}
def to_jsonable(value):
if isinstance(value, dict):
return {str(k): to_jsonable(v) for k, v in value.items()}
if hasattr(value, "detach"):
return value.detach().cpu().tolist()
if isinstance(value, (list, tuple)):
return [to_jsonable(v) for v in value]
return value
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Depth Anything V2 adapter for the Real2Sim hybrid backend.
The adapter writes a normalized PGM/PNG depth map that the built-in asset
generator can fuse with its geometry prior. The default model is the small
Transformers checkpoint so it can run on the Wuying 12 GB GPU.
"""
from __future__ import annotations
import argparse
import os
import sys
from pathlib import Path
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.image_io import write_gray
def main() -> int:
parser = argparse.ArgumentParser(description="Run Depth Anything V2 monocular depth for Real2Sim")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument(
"--model",
default=os.environ.get("REAL2SIM_DEPTH_MODEL", "depth-anything/Depth-Anything-V2-Small-hf"),
help="Hugging Face depth-estimation model id",
)
parser.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"])
parser.add_argument("--invert", dest="invert", action="store_true", help="Invert relative depth before writing")
parser.add_argument("--no-invert", dest="invert", action="store_false")
parser.set_defaults(invert=True)
args = parser.parse_args()
depth = estimate_depth(args.image, args.model, args.device)
if args.invert:
depth = 1.0 - depth
write_gray(args.output, depth)
return 0
def estimate_depth(image_path: Path, model_id: str, device: str) -> np.ndarray:
try:
import torch
from PIL import Image
from transformers import pipeline
except Exception as exc: # pragma: no cover - external runtime guard
raise SystemExit(
"Depth Anything requires torch, pillow, and transformers. "
"Install them in the Real2Sim environment before enabling depth_estimation."
) from exc
image = Image.open(image_path).convert("RGB")
device_index = -1
if device == "cuda" or (device == "auto" and torch.cuda.is_available()):
device_index = 0
pipe = pipeline(task="depth-estimation", model=model_id, device=device_index)
result = pipe(image)
raw_depth = result.get("predicted_depth")
if raw_depth is not None:
depth = raw_depth.detach().float().cpu().numpy()
depth = np.squeeze(depth)
else:
depth = np.asarray(result["depth"], dtype=np.float32)
if depth.ndim == 3:
depth = depth[:, :, 0]
return normalize_depth(depth)
def normalize_depth(depth: np.ndarray) -> np.ndarray:
depth = depth.astype(np.float32)
lo = float(np.percentile(depth, 1.0))
hi = float(np.percentile(depth, 99.0))
if hi <= lo:
return np.zeros_like(depth, dtype=np.float32)
return np.clip((depth - lo) / (hi - lo), 0.0, 1.0).astype(np.float32)
if __name__ == "__main__":
raise SystemExit(main())

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""GroundingDINO detection adapter for Real2Sim.
This uses the Hugging Face GroundingDINO interface used by the official
Grounded-SAM-2 demos. It writes the Real2Sim detections.json contract.
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.contracts import write_json
def main() -> int:
parser = argparse.ArgumentParser(description="Run GroundingDINO via Hugging Face and write detections.json")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--prompt", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--model-id", default="IDEA-Research/grounding-dino-tiny")
parser.add_argument("--box-threshold", type=float, default=0.25)
parser.add_argument("--text-threshold", type=float, default=0.25)
parser.add_argument("--nms-threshold", type=float, default=0.6)
parser.add_argument("--max-detections", type=int, default=24)
parser.add_argument("--device", default="auto", choices=["auto", "cuda", "cpu"])
args = parser.parse_args()
import torch
from PIL import Image
from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor
device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device
if device == "auto":
device = "cpu"
image = Image.open(args.image).convert("RGB")
processor = AutoProcessor.from_pretrained(args.model_id)
model = AutoModelForZeroShotObjectDetection.from_pretrained(args.model_id).to(device)
detections = []
queries = prompt_queries(args.prompt)
for prompt_index, prompt in enumerate(queries):
inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)
with torch.no_grad():
outputs = model(**inputs)
result = processor.post_process_grounded_object_detection(
outputs,
inputs.input_ids,
threshold=args.box_threshold,
text_threshold=args.text_threshold,
target_sizes=[image.size[::-1]],
)[0]
boxes = result.get("boxes", [])
scores = result.get("scores", [])
labels = result.get("labels", [])
for idx, box in enumerate(boxes):
label = labels[idx] if idx < len(labels) else prompt.rstrip(".")
score = float(scores[idx].detach().cpu().item()) if idx < len(scores) else None
bbox = [float(v) for v in box.detach().cpu().tolist()]
detections.append(
{
"id": f"det_{prompt_index:03d}_{idx:03d}",
"label": str(label).strip().rstrip(".") or prompt.rstrip("."),
"score": score,
"bbox_xyxy": bbox,
"prompt_query": prompt,
"prompt_index": prompt_index,
}
)
detections = nms_detections(detections, args.nms_threshold)[: max(0, args.max_detections)]
write_json(
args.output,
{
"detections": detections,
"metadata": {
"backend": "groundingdino_hf",
"model_id": args.model_id,
"prompt": args.prompt,
"prompt_queries": queries,
"detection_scheme": "prompt_wise_independent",
"box_threshold": args.box_threshold,
"text_threshold": args.text_threshold,
"nms_threshold": args.nms_threshold,
"max_detections": args.max_detections,
},
},
)
return 0
def normalize_prompt(prompt: str) -> str:
text = " ".join(prompt.strip().lower().split())
if not text:
text = "object"
return text if text.endswith(".") else f"{text}."
def prompt_queries(prompt: str) -> list[str]:
parts = [part.strip() for part in re.split(r"[.,;\n]+", prompt) if part.strip()]
if not parts:
parts = ["object"]
return [normalize_prompt(part) for part in parts]
def nms_detections(detections: list[dict], threshold: float) -> list[dict]:
pending = sorted(detections, key=lambda item: float(item.get("score") or 0.0), reverse=True)
kept: list[dict] = []
while pending:
current = pending.pop(0)
kept.append(current)
pending = [
candidate
for candidate in pending
if current.get("label") != candidate.get("label")
or iou(current["bbox_xyxy"], candidate["bbox_xyxy"]) < threshold
]
for idx, item in enumerate(kept):
item["id"] = f"det_{idx:03d}"
return kept
def iou(a: list[float], b: list[float]) -> float:
ax0, ay0, ax1, ay1 = a
bx0, by0, bx1, by1 = b
ix0, iy0 = max(ax0, bx0), max(ay0, by0)
ix1, iy1 = min(ax1, bx1), min(ay1, by1)
iw, ih = max(0.0, ix1 - ix0), max(0.0, iy1 - iy0)
inter = iw * ih
area_a = max(0.0, ax1 - ax0) * max(0.0, ay1 - ay0)
area_b = max(0.0, bx1 - bx0) * max(0.0, by1 - by0)
union = area_a + area_b - inter
return inter / union if union > 0 else 0.0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,328 @@
#!/usr/bin/env python3
"""Paper-style visual instance refinement for Real2Sim.
The GS-Playground paper avoids semantic-similarity de-duplication after
open-vocabulary detection. This stage uses only visual mask criteria and a
composite detection/segmentation confidence score.
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from typing import Any
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.contracts import read_json, write_json
from real2sim.image_io import read_gray, read_rgb, write_gray
def main() -> int:
parser = argparse.ArgumentParser(description="Refine SAM/SAM2 instances using visual de-duplication")
parser.add_argument("--instances", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--image", type=Path, default=None, help="Optional source RGB image for accessory mask recovery")
parser.add_argument("--mask-iou-threshold", type=float, default=0.75)
parser.add_argument("--inclusion-threshold", type=float, default=0.92)
parser.add_argument("--boundary-overlap-threshold", type=float, default=0.85)
parser.add_argument("--merge-saturated-accessories", action="store_true")
parser.add_argument("--accessory-search-margin-ratio", type=float, default=0.25)
parser.add_argument("--accessory-saturation-threshold", type=float, default=0.45)
parser.add_argument("--accessory-contact-passes", type=int, default=4)
parser.add_argument("--accessory-max-area-growth", type=float, default=0.15)
parser.add_argument("--accessory-min-component-area-ratio", type=float, default=0.001)
parser.add_argument("--accessory-max-components", type=int, default=8)
args = parser.parse_args()
raw = read_json(args.instances)
entries = raw.get("instances", raw) if isinstance(raw, dict) else raw
image = read_rgb(args.image) if args.image is not None and args.image.exists() else None
refined = refine_instance_entries(
entries,
args.instances.parent,
image=image,
merge_saturated_accessories=args.merge_saturated_accessories,
mask_iou_threshold=args.mask_iou_threshold,
inclusion_threshold=args.inclusion_threshold,
boundary_overlap_threshold=args.boundary_overlap_threshold,
accessory_search_margin_ratio=args.accessory_search_margin_ratio,
accessory_saturation_threshold=args.accessory_saturation_threshold,
accessory_contact_passes=args.accessory_contact_passes,
accessory_max_area_growth=args.accessory_max_area_growth,
accessory_min_component_area_ratio=args.accessory_min_component_area_ratio,
accessory_max_components=args.accessory_max_components,
)
metadata = dict(raw.get("metadata", {})) if isinstance(raw, dict) else {}
metadata["instance_refinement"] = {
"backend": "visual_mask_dedup",
"input_instances": len(entries),
"output_instances": len(refined),
"mask_iou_threshold": args.mask_iou_threshold,
"inclusion_threshold": args.inclusion_threshold,
"boundary_overlap_threshold": args.boundary_overlap_threshold,
"merge_saturated_accessories": args.merge_saturated_accessories,
"accessory_search_margin_ratio": args.accessory_search_margin_ratio,
"accessory_saturation_threshold": args.accessory_saturation_threshold,
"accessory_contact_passes": args.accessory_contact_passes,
"accessory_max_area_growth": args.accessory_max_area_growth,
"accessory_min_component_area_ratio": args.accessory_min_component_area_ratio,
"accessory_max_components": args.accessory_max_components,
"accessory_pixels_added": sum(int(entry.get("accessory_pixels_added", 0)) for entry in refined),
}
write_json(args.output, {"instances": refined, "metadata": metadata})
return 0
def refine_instance_entries(
entries: list[dict[str, Any]],
base_dir: Path,
*,
image: np.ndarray | None = None,
merge_saturated_accessories: bool = False,
mask_iou_threshold: float = 0.75,
inclusion_threshold: float = 0.92,
boundary_overlap_threshold: float = 0.85,
accessory_search_margin_ratio: float = 0.25,
accessory_saturation_threshold: float = 0.45,
accessory_contact_passes: int = 4,
accessory_max_area_growth: float = 0.15,
accessory_min_component_area_ratio: float = 0.001,
accessory_max_components: int = 8,
) -> list[dict[str, Any]]:
prepared = []
for index, entry in enumerate(entries):
mask_path = entry.get("mask_path", entry.get("mask"))
if not mask_path:
continue
path = Path(str(mask_path))
path = path if path.is_absolute() else (base_dir / path).resolve()
if not path.exists():
continue
mask = read_gray(path) > 0.5
if merge_saturated_accessories and image is not None and image.shape[:2] == mask.shape:
mask, merge_event = merge_saturated_accessories_into_mask(
mask,
image,
bbox_xyxy=entry.get("bbox_xyxy"),
search_margin_ratio=accessory_search_margin_ratio,
saturation_threshold=accessory_saturation_threshold,
contact_passes=accessory_contact_passes,
max_area_growth=accessory_max_area_growth,
min_component_area_ratio=accessory_min_component_area_ratio,
max_components=accessory_max_components,
)
if merge_event["added_pixels"] > 0:
write_gray(path, mask.astype(np.float32))
entry = dict(entry)
entry["accessory_pixels_added"] = merge_event["added_pixels"]
entry["accessory_components_added"] = merge_event["components_added"]
score = composite_score(entry)
item = dict(entry)
item["composite_score"] = score
item["_input_index"] = index
prepared.append((item, mask))
prepared.sort(key=lambda pair: (-float(pair[0]["composite_score"]), int(pair[0]["_input_index"])))
kept: list[tuple[dict[str, Any], np.ndarray]] = []
for item, mask in prepared:
duplicate = False
for kept_item, kept_mask in kept:
if is_visual_duplicate(
mask,
kept_mask,
mask_iou_threshold=mask_iou_threshold,
inclusion_threshold=inclusion_threshold,
boundary_overlap_threshold=boundary_overlap_threshold,
):
duplicate = True
break
if not duplicate:
kept.append((item, mask))
kept.sort(key=lambda pair: int(pair[0]["_input_index"]))
output = []
for idx, (item, _) in enumerate(kept):
cleaned = dict(item)
cleaned.pop("_input_index", None)
cleaned["id"] = str(cleaned.get("id", cleaned.get("instance_id", f"inst_{idx:03d}")))
output.append(cleaned)
return output
def composite_score(entry: dict[str, Any]) -> float:
detection = entry.get("detection_score", entry.get("score", 0.0))
segmentation = entry.get("segmentation_score", entry.get("mask_score", detection))
detection = 0.0 if detection is None else float(detection)
segmentation = 0.0 if segmentation is None else float(segmentation)
return 0.5 * detection + 0.5 * segmentation
def is_visual_duplicate(
mask: np.ndarray,
reference: np.ndarray,
*,
mask_iou_threshold: float,
inclusion_threshold: float,
boundary_overlap_threshold: float,
) -> bool:
if mask.shape != reference.shape:
return False
intersection = int((mask & reference).sum())
area = int(mask.sum())
reference_area = int(reference.sum())
if area == 0 or reference_area == 0:
return False
union = area + reference_area - intersection
if union > 0 and intersection / union >= mask_iou_threshold:
return True
smaller = mask if area <= reference_area else reference
larger = reference if area <= reference_area else mask
inclusion = int((smaller & larger).sum()) / float(int(smaller.sum()))
boundary_overlap = int((boundary(smaller) & larger).sum()) / max(1, int(boundary(smaller).sum()))
return inclusion >= inclusion_threshold and boundary_overlap >= boundary_overlap_threshold
def boundary(mask: np.ndarray) -> np.ndarray:
eroded = mask.copy()
padded = np.pad(mask, 1, mode="constant", constant_values=False)
eroded &= padded[:-2, 1:-1]
eroded &= padded[2:, 1:-1]
eroded &= padded[1:-1, :-2]
eroded &= padded[1:-1, 2:]
return mask & ~eroded
def merge_saturated_accessories_into_mask(
mask: np.ndarray,
image: np.ndarray,
*,
bbox_xyxy: Any | None = None,
search_margin_ratio: float = 0.25,
saturation_threshold: float = 0.45,
contact_passes: int = 4,
max_area_growth: float = 0.15,
min_component_area_ratio: float = 0.001,
max_components: int = 8,
) -> tuple[np.ndarray, dict[str, int]]:
base = np.asarray(mask, dtype=bool)
if not base.any():
return base, {"added_pixels": 0, "components_added": 0}
search = expanded_search_region(base, bbox_xyxy, margin_ratio=search_margin_ratio)
rgb = np.asarray(image, dtype=np.float32)
if rgb.max(initial=0.0) > 1.5:
rgb = rgb / 255.0
saturation = rgb_saturation(rgb)
candidates = (saturation >= saturation_threshold) & search & ~base
if not candidates.any():
return base, {"added_pixels": 0, "components_added": 0}
contact = dilate(base, max(0, int(contact_passes)))
max_added = max(1, int(base.sum() * max(0.0, max_area_growth)))
min_component_area = max(1, int(np.ceil(base.sum() * max(0.0, min_component_area_ratio))))
merged = base.copy()
added_pixels = 0
components_added = 0
components = connected_components(candidates)
components.sort(key=lambda comp: int(comp.sum()), reverse=True)
for component in components:
if max_components > 0 and components_added >= max_components:
break
area = int(component.sum())
if area < min_component_area or added_pixels + area > max_added:
continue
if not (component & contact).any():
continue
merged |= component
contact = dilate(merged, max(0, int(contact_passes)))
added_pixels += area
components_added += 1
return merged, {"added_pixels": added_pixels, "components_added": components_added}
def expanded_search_region(mask: np.ndarray, bbox_xyxy: Any | None, *, margin_ratio: float) -> np.ndarray:
h, w = mask.shape
y0, y1, x0, x1 = mask_bounds(mask)
if bbox_xyxy is not None:
try:
bx0, by0, bx1, by1 = [float(v) for v in bbox_xyxy]
x0 = min(x0, int(np.floor(bx0)))
y0 = min(y0, int(np.floor(by0)))
x1 = max(x1, int(np.ceil(bx1)))
y1 = max(y1, int(np.ceil(by1)))
except (TypeError, ValueError):
pass
box_w = max(1, x1 - x0 + 1)
box_h = max(1, y1 - y0 + 1)
margin_x = max(1, int(round(box_w * max(0.0, margin_ratio))))
margin_y = max(1, int(round(box_h * max(0.0, margin_ratio))))
x0 = max(0, x0 - margin_x)
x1 = min(w - 1, x1 + margin_x)
y0 = max(0, y0 - margin_y)
y1 = min(h - 1, y1 + margin_y)
region = np.zeros((h, w), dtype=bool)
region[y0 : y1 + 1, x0 : x1 + 1] = True
return region
def mask_bounds(mask: np.ndarray) -> tuple[int, int, int, int]:
ys, xs = np.where(mask)
if ys.size == 0:
return 0, 0, 0, 0
return int(ys.min()), int(ys.max()), int(xs.min()), int(xs.max())
def rgb_saturation(rgb: np.ndarray) -> np.ndarray:
clipped = np.clip(rgb[:, :, :3], 0.0, 1.0)
maxc = clipped.max(axis=2)
minc = clipped.min(axis=2)
return (maxc - minc) / np.maximum(maxc, 1e-6)
def connected_components(mask: np.ndarray) -> list[np.ndarray]:
h, w = mask.shape
visited = np.zeros_like(mask, dtype=bool)
components: list[np.ndarray] = []
ys, xs = np.where(mask)
for sy, sx in zip(ys.tolist(), xs.tolist()):
if visited[sy, sx] or not mask[sy, sx]:
continue
stack = [(sy, sx)]
visited[sy, sx] = True
pixels = []
while stack:
y, x = stack.pop()
pixels.append((y, x))
for ny in range(y - 1, y + 2):
for nx in range(x - 1, x + 2):
if ny == y and nx == x:
continue
if 0 <= ny < h and 0 <= nx < w and mask[ny, nx] and not visited[ny, nx]:
visited[ny, nx] = True
stack.append((ny, nx))
component = np.zeros_like(mask, dtype=bool)
for y, x in pixels:
component[y, x] = True
components.append(component)
return components
def dilate(mask: np.ndarray, passes: int) -> np.ndarray:
out = np.asarray(mask, dtype=bool).copy()
for _ in range(max(0, passes)):
padded = np.pad(out, 1, mode="constant", constant_values=False)
expanded = out.copy()
for dy in range(3):
for dx in range(3):
expanded |= padded[dy : dy + out.shape[0], dx : dx + out.shape[1]]
out = expanded
return out
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,225 @@
#!/usr/bin/env python3
"""Sequential LaMa inpainting adapter for Real2Sim.
Preferred runtime is the lightweight simple-lama-inpainting package, which is
listed by the official LaMa repository as a third-party app. This adapter keeps
the Real2Sim contract stable: image + instances.json -> background image.
"""
from __future__ import annotations
import argparse
import os
import subprocess
import sys
from pathlib import Path
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.commands import format_command
from real2sim.contracts import load_instances, write_json
from real2sim.image_io import read_gray, read_rgb, write_gray, write_rgb
def main() -> int:
parser = argparse.ArgumentParser(description="Run LaMa object removal from Real2Sim instance masks")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--instances", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--dilate", type=int, default=7)
parser.add_argument("--prompt", default="object")
parser.add_argument("--work-dir", type=Path, default=None)
parser.add_argument("--redetect-command", default="")
parser.add_argument("--segment-command", default="")
parser.add_argument("--redetect-command-env", default="")
parser.add_argument("--segment-command-env", default="")
parser.add_argument("--require-redetect-loop", action="store_true")
parser.add_argument("--max-recovery-passes", type=int, default=1)
parser.add_argument("--max-area-growth", type=float, default=0.5)
args = parser.parse_args()
instances = load_instances(args.instances)
image = read_rgb(args.image)
redetect_command = args.redetect_command or (os.environ.get(args.redetect_command_env, "") if args.redetect_command_env else "")
segment_command = args.segment_command or (os.environ.get(args.segment_command_env, "") if args.segment_command_env else "")
require_recovery_commands(redetect_command, segment_command, strict=args.require_redetect_loop)
try:
from PIL import Image
from simple_lama_inpainting import SimpleLama
except Exception as exc:
raise SystemExit(
"simple-lama-inpainting is not installed. Install it in the external "
"model environment or use the built-in backend."
) from exc
lama = SimpleLama()
current = image
steps = []
recovery_events = []
work_dir = args.work_dir or args.output.parent
instance_masks = {
instance.instance_id: read_gray(instance.mask_path) > 0.5
for instance in instances
if instance.mask_path and instance.mask_path.exists()
}
ordered = sorted(
[instance for instance in instances if instance.mask_path],
key=lambda inst: (-(inst.score if inst.score is not None else 0.0), inst.instance_id),
)
for instance in ordered:
mask = read_gray(instance.mask_path) > 0.5
if mask.shape != current.shape[:2]:
continue
expanded = dilate(mask, args.dilate)
pil_image = Image.fromarray((np.clip(current, 0, 1) * 255).astype(np.uint8))
pil_mask = Image.fromarray((expanded.astype(np.uint8) * 255), mode="L")
result = lama(pil_image, pil_mask)
current = np.asarray(result.convert("RGB"), dtype=np.float32) / 255.0
if redetect_command and segment_command:
recovery_events.extend(
recover_occluded_regions(
current,
instances=instances,
instance_masks=instance_masks,
instances_json=args.instances,
prompt=args.prompt,
work_dir=work_dir,
pass_index=len(steps),
redetect_command=redetect_command,
segment_command=segment_command,
max_area_growth=args.max_area_growth,
)
)
steps.append(
{
"instance_id": instance.instance_id,
"label": instance.label,
"mask_pixels": int(mask.sum()),
"expanded_mask_pixels": int(expanded.sum()),
}
)
write_rgb(args.output, current)
write_json(
args.output.with_suffix(args.output.suffix + ".json"),
{
"backend": "simple_lama_sequential",
"strategy": "one_object_at_a_time_mask_expansion",
"steps": steps,
"recovery_events": recovery_events,
"redetect_loop": bool(redetect_command and segment_command),
},
)
return 0
def require_recovery_commands(redetect_command: str, segment_command: str, *, strict: bool) -> None:
if strict and (not redetect_command or not segment_command):
raise SystemExit(
"paper-strict sequential inpainting requires both a re-detection command "
"and a segmentation command for newly exposed region recovery"
)
def recover_occluded_regions(
image: np.ndarray,
*,
instances,
instance_masks: dict[str, np.ndarray],
instances_json: Path,
prompt: str,
work_dir: Path,
pass_index: int,
redetect_command,
segment_command,
max_area_growth: float,
) -> list[dict]:
recovery_dir = work_dir / "03_inpainting" / "recovery" / f"pass_{pass_index:03d}"
recovery_dir.mkdir(parents=True, exist_ok=True)
image_path = recovery_dir / "image.ppm"
detections_json = recovery_dir / "detections.json"
masks_dir = recovery_dir / "masks"
recovered_instances_json = recovery_dir / "instances.json"
write_rgb(image_path, image)
base_context = {
"python": sys.executable,
"image": image_path,
"input": image_path,
"prompt": prompt,
"detections_json": detections_json,
"instances_json": recovered_instances_json,
"masks_dir": masks_dir,
}
run_nested_command(redetect_command, base_context)
run_nested_command(segment_command, base_context)
existing = {instance.instance_id: instance for instance in instances}
events = []
for candidate in load_instances(recovered_instances_json):
if candidate.mask_path is None or not candidate.mask_path.exists():
continue
candidate_mask = read_gray(candidate.mask_path) > 0.5
for instance in instances:
if instance.label != candidate.label or instance.mask_path is None:
continue
base_mask = instance_masks.get(instance.instance_id)
if base_mask is None:
continue
merged = merge_recovered_mask(base_mask, candidate_mask, same_label=True, max_area_growth=max_area_growth)
if int(merged.sum()) == int(base_mask.sum()):
continue
write_gray(instance.mask_path, merged.astype(np.float32))
instance_masks[instance.instance_id] = merged
events.append(
{
"target_instance_id": instance.instance_id,
"candidate_instance_id": candidate.instance_id,
"label": instance.label,
"old_mask_pixels": int(base_mask.sum()),
"new_mask_pixels": int(merged.sum()),
}
)
break
return events
def run_nested_command(command, context: dict) -> None:
argv = format_command(command, context)
completed = subprocess.run(argv, cwd=REPO_ROOT, check=False)
if completed.returncode != 0:
raise SystemExit(f"nested recovery command failed with exit code {completed.returncode}: {argv}")
def merge_recovered_mask(base: np.ndarray, candidate: np.ndarray, *, same_label: bool, max_area_growth: float) -> np.ndarray:
if not same_label or base.shape != candidate.shape:
return base
new_region = candidate & ~base
if not new_region.any():
return base
if int(new_region.sum()) > int(base.sum()) * max_area_growth:
return base
if not (dilate(base, 1) & new_region).any():
return base
return base | new_region
def dilate(mask: np.ndarray, passes: int) -> np.ndarray:
out = mask.copy()
for _ in range(max(0, passes)):
padded = np.pad(out, 1, mode="constant", constant_values=False)
out = (
padded[1:-1, 1:-1]
| padded[:-2, 1:-1]
| padded[2:, 1:-1]
| padded[1:-1, :-2]
| padded[1:-1, 2:]
)
return out
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""SAM2 segmentation adapter for Real2Sim.
Consumes detections.json boxes and writes instances.json plus per-instance PGM
masks. This uses Meta's official SAM2 image predictor API.
"""
from __future__ import annotations
import argparse
import os
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.contracts import read_json, write_json
from real2sim.image_io import write_gray
def main() -> int:
parser = argparse.ArgumentParser(description="Run SAM2 masks from GroundingDINO boxes")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--detections", type=Path, required=True)
parser.add_argument("--masks-dir", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--checkpoint", default=os.environ.get("REAL2SIM_SAM2_CHECKPOINT", ""))
parser.add_argument("--model-cfg", default=os.environ.get("REAL2SIM_SAM2_MODEL_CFG", "configs/sam2.1/sam2.1_hiera_l.yaml"))
parser.add_argument("--device", default="auto", choices=["auto", "cuda", "cpu"])
args = parser.parse_args()
if not args.checkpoint:
raise SystemExit("SAM2 checkpoint is required via --checkpoint or REAL2SIM_SAM2_CHECKPOINT")
import numpy as np
import torch
from PIL import Image
from sam2.build_sam import build_sam2
from sam2.sam2_image_predictor import SAM2ImagePredictor
detections_raw = read_json(args.detections)
detections = detections_raw.get("detections", detections_raw) if isinstance(detections_raw, dict) else detections_raw
if not detections:
write_json(args.output, {"instances": [], "metadata": {"backend": "sam2"}})
return 0
boxes = np.array([entry.get("bbox_xyxy", entry.get("bbox")) for entry in detections], dtype=np.float32)
device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device
if device == "auto":
device = "cpu"
image = np.array(Image.open(args.image).convert("RGB"))
model = build_sam2(args.model_cfg, args.checkpoint, device=device)
predictor = SAM2ImagePredictor(model)
args.masks_dir.mkdir(parents=True, exist_ok=True)
with torch.inference_mode():
if device == "cuda":
with torch.autocast("cuda", dtype=torch.bfloat16):
masks, scores, _ = predict_masks(predictor, image, boxes)
else:
masks, scores, _ = predict_masks(predictor, image, boxes)
masks = normalize_masks(masks)
instances = []
for idx, (detection, mask) in enumerate(zip(detections, masks)):
instance_id = str(detection.get("id", f"det_{idx:03d}"))
mask_path = args.masks_dir / f"{safe_name(instance_id)}.pgm"
write_gray(mask_path, mask.astype(np.float32))
segmentation_score = None
if scores is not None and idx < len(scores):
segmentation_score = float(np.asarray(scores[idx]).reshape(-1)[0])
instances.append(
instance_record(
detection,
mask_path=mask_path,
output_parent=args.output.parent,
segmentation_score=segmentation_score,
)
)
write_json(args.output, {"instances": instances, "metadata": {"backend": "sam2", "model_cfg": args.model_cfg}})
return 0
def predict_masks(predictor, image, boxes):
predictor.set_image(image)
return predictor.predict(point_coords=None, point_labels=None, box=boxes, multimask_output=False)
def normalize_masks(masks):
import numpy as np
masks = np.asarray(masks)
if masks.ndim == 4:
masks = masks[:, 0]
if masks.ndim == 2:
masks = masks[None]
return masks > 0
def instance_record(detection: dict, *, mask_path: Path, output_parent: Path, segmentation_score: float | None) -> dict:
instance_id = str(detection.get("id", "instance"))
detection_score = detection.get("score")
score = float(detection_score) if detection_score is not None else segmentation_score
record = {
"id": instance_id,
"label": str(detection.get("label", instance_id)),
"score": score,
"bbox_xyxy": detection.get("bbox_xyxy", detection.get("bbox")),
"mask_path": mask_path.relative_to(output_parent).as_posix(),
}
if detection_score is not None:
record["detection_score"] = float(detection_score)
if segmentation_score is not None:
record["segmentation_score"] = float(segmentation_score)
for key in ("prompt_index", "prompt_query"):
if key in detection:
record[key] = detection[key]
return record
def safe_name(value: str) -> str:
return "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in value).strip("_") or "instance"
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,897 @@
#!/usr/bin/env python3
"""SAM 3D Objects adapter for one Real2Sim object.
This follows Meta's public quick-start shape:
from inference import Inference, load_image, load_single_mask
output = inference(image, mask)
output["gs"].save_ply("splat.ply")
The script intentionally fails with a clear message if the gated checkpoints
or SAM 3D environment are not available.
"""
from __future__ import annotations
import argparse
import importlib
import inspect
import json
import os
import struct
import sys
from pathlib import Path
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.contracts import Pose, read_json, write_json
from real2sim.image_io import read_gray, write_gray
class PreparedSam3DInputs:
def __init__(self, image_path: Path, mask_path: Path, mask: np.ndarray, image_size: tuple[int, int], metadata: dict):
self.image_path = image_path
self.mask_path = mask_path
self.mask = mask
self.image_size = image_size
self.metadata = metadata
class PlyVertexTable:
def __init__(
self,
*,
header_lines: list[str],
body_start: int,
fmt: str,
vertex_count: int,
property_names: list[str],
property_types: list[str],
array: np.ndarray,
):
self.header_lines = header_lines
self.body_start = body_start
self.fmt = fmt
self.vertex_count = vertex_count
self.property_names = property_names
self.property_types = property_types
self.array = array
def main() -> int:
parser = argparse.ArgumentParser(description="Run SAM 3D Objects for one detected object")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--mask", type=Path, required=True)
parser.add_argument("--instance", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--ply", type=Path, required=True)
parser.add_argument("--mesh", type=Path, required=True)
parser.add_argument("--pose", type=Path, required=True)
parser.add_argument("--cameras", type=Path, default=None, help="AnySplat cameras JSON used to match image preprocessing")
parser.add_argument("--match-camera-preprocess", action="store_true", help="Run SAM-3D on the same aspect-preserving center crop used by AnySplat")
parser.add_argument("--object-centric-preprocess", action="store_true", help="Crop around the instance mask before SAM-3D preprocessing")
parser.add_argument("--object-preprocess-size", type=int, default=0, help="Override SAM-3D object input size with a square image")
parser.add_argument("--object-crop-margin", type=float, default=0.12, help="Relative padding around the instance mask for object-centric preprocessing")
parser.add_argument("--remove-video-overlay", action="store_true", help="Inpaint a central video play overlay before SAM-3D inference")
parser.add_argument("--canonicalize-gaussian-to-mesh", action="store_true", help="Rotate SAM-3D Gaussian PLY into the mesh canonical frame")
parser.add_argument("--texture-baking", action="store_true", help="Use SAM-3D's textured GLB postprocess path for the exported preview mesh")
parser.add_argument("--texture-render-backend", choices=("auto", "inria", "gsplat"), default="gsplat")
parser.add_argument("--texture-baking-engine", choices=("nvdiffrast", "pytorch3d"), default="pytorch3d")
parser.add_argument("--sam3d-root", type=Path, default=os.environ.get("REAL2SIM_SAM3D_ROOT"))
parser.add_argument("--config", type=Path, default=os.environ.get("REAL2SIM_SAM3D_CONFIG"))
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--allow-fallback-mesh", action="store_true", help="Write a collision proxy if SAM 3D returns no exportable mesh")
args = parser.parse_args()
if args.sam3d_root is None:
raise SystemExit("SAM 3D root is required via --sam3d-root or REAL2SIM_SAM3D_ROOT")
if args.config is None:
args.config = args.sam3d_root / "checkpoints" / "hf" / "pipeline.yaml"
if not args.config.exists():
raise SystemExit(f"SAM 3D config/checkpoints are not available: {args.config}")
sys.path.insert(0, (args.sam3d_root / "notebook").as_posix())
try:
from inference import Inference, load_image
except Exception as exc:
raise SystemExit(f"failed to import SAM 3D inference from {args.sam3d_root}: {exc}") from exc
import torch
args.output_dir.mkdir(parents=True, exist_ok=True)
image_path = args.image
mask_np = (read_gray(args.mask) > 0.5).astype(np.uint8)
preprocess_metadata = None
overlay_metadata = None
if args.match_camera_preprocess:
if args.cameras is None:
raise SystemExit("--match-camera-preprocess requires --cameras")
target_shape = None
if args.object_preprocess_size > 0:
target_shape = (int(args.object_preprocess_size), int(args.object_preprocess_size))
prepared = prepare_camera_preprocessed_inputs(
args.image,
args.mask,
args.cameras,
args.output_dir / "camera_preprocess",
object_centric=args.object_centric_preprocess,
target_shape=target_shape,
object_margin_ratio=args.object_crop_margin,
)
image_path = prepared.image_path
mask_np = prepared.mask.astype(np.uint8)
preprocess_metadata = prepared.metadata
if args.remove_video_overlay:
image_path, overlay_metadata = remove_center_video_overlay(image_path, mask_np > 0, args.output_dir / "video_overlay_removal")
image = load_image(image_path.as_posix())
inference = Inference(args.config.as_posix(), compile=False)
with torch.inference_mode():
output = run_sam3d_inference(
inference,
image,
mask_np,
seed=args.seed,
texture_baking=args.texture_baking,
texture_render_backend=args.texture_render_backend,
texture_baking_engine=args.texture_baking_engine,
)
save_gaussian(output, args.ply)
wrote_mesh = export_mesh(output, args.mesh, prefer_textured=args.texture_baking)
if not wrote_mesh and args.allow_fallback_mesh:
write_bbox_mesh_from_mask(args.mesh, mask_np)
wrote_mesh = True
if not wrote_mesh:
raise SystemExit(
"SAM 3D output did not include an exportable mesh. "
"Use --allow-fallback-mesh only for collision-proxy debugging. "
f"Output keys: {sorted(output.keys())}"
)
canonicalization_metadata = None
if args.canonicalize_gaussian_to_mesh:
canonicalization_metadata = canonicalize_gaussian_to_mesh_frame(args.ply, args.mesh)
pose = Pose()
layout = output.get("layout") or output.get("pose") or output
if isinstance(layout, dict):
pose = pose_from_layout(layout)
write_json(args.pose, pose.to_json())
metadata = {
"backend": "sam3d_objects",
"source_instance": read_json(args.instance),
"sam3d_config": args.config.as_posix(),
"texture_baking": bool(args.texture_baking),
"texture_render_backend": args.texture_render_backend,
"texture_baking_engine": args.texture_baking_engine,
}
if preprocess_metadata is not None:
metadata["camera_preprocess"] = preprocess_metadata
if overlay_metadata is not None:
metadata["video_overlay_removal"] = overlay_metadata
if canonicalization_metadata is not None:
metadata["gaussian_canonicalization"] = canonicalization_metadata
write_json(args.output_dir / "sam3d_metadata.json", metadata)
return 0
def prepare_camera_preprocessed_inputs(
image_path: Path,
mask_path: Path,
cameras_path: Path,
output_dir: Path,
*,
object_centric: bool = False,
target_shape: tuple[int, int] | None = None,
object_margin_ratio: float = 0.12,
) -> PreparedSam3DInputs:
cameras = read_json(cameras_path)
target_h, target_w = target_shape or camera_image_shape(cameras)
from PIL import Image
image = Image.open(image_path).convert("RGB")
mask = read_gray(mask_path) > 0.5
if object_centric and mask.any():
processed_image, processed_mask, crop_meta = object_centric_resize_pad_image_and_mask(
image,
mask,
(target_h, target_w),
margin_ratio=object_margin_ratio,
resample=Image.BICUBIC,
)
else:
processed_image, crop_meta = cover_resize_center_crop_image(image, (target_h, target_w), resample=Image.BICUBIC)
processed_mask = cover_resize_center_crop_mask(mask, (target_h, target_w), crop_meta)
crop_meta = {"preprocess_mode": "camera_center_crop", **crop_meta}
output_dir.mkdir(parents=True, exist_ok=True)
processed_image_path = output_dir / "sam3d_camera_image.png"
processed_mask_path = output_dir / "sam3d_camera_mask.pgm"
processed_image.save(processed_image_path)
write_gray(processed_mask_path, processed_mask.astype(np.float32))
metadata = {
"source_image": image_path.as_posix(),
"source_mask": mask_path.as_posix(),
"cameras": cameras_path.as_posix(),
"source_size": [int(image.height), int(image.width)],
"target_size": [int(target_h), int(target_w)],
**crop_meta,
"image": processed_image_path.as_posix(),
"mask": processed_mask_path.as_posix(),
"mask_pixels": int(processed_mask.sum()),
}
return PreparedSam3DInputs(
image_path=processed_image_path,
mask_path=processed_mask_path,
mask=processed_mask,
image_size=(target_h, target_w),
metadata=metadata,
)
def camera_image_shape(cameras: dict) -> tuple[int, int]:
shape = cameras.get("image_shape") if isinstance(cameras, dict) else None
if isinstance(shape, (list, tuple)) and len(shape) >= 2:
return (int(shape[0]), int(shape[1]))
intrinsics = cameras.get("intrinsics", {}) if isinstance(cameras, dict) else {}
height = int(intrinsics.get("height", 448))
width = int(intrinsics.get("width", 448))
return (height, width)
def cover_resize_center_crop_image(image, shape: tuple[int, int], *, resample) -> tuple:
dst_h, dst_w = shape
src_w, src_h = image.size
scale = max(dst_w / float(src_w), dst_h / float(src_h))
resized_w = max(dst_w, int(src_w * scale))
resized_h = max(dst_h, int(src_h * scale))
resized = image.resize((resized_w, resized_h), resample)
left = max(0, (resized_w - dst_w) // 2)
top = max(0, (resized_h - dst_h) // 2)
crop = resized.crop((left, top, left + dst_w, top + dst_h))
return crop, {
"resized_size": [int(resized_h), int(resized_w)],
"crop_xyxy": [int(left), int(top), int(left + dst_w), int(top + dst_h)],
"scale": float(scale),
}
def cover_resize_center_crop_mask(mask: np.ndarray, shape: tuple[int, int], crop_meta: dict) -> np.ndarray:
from PIL import Image
dst_h, dst_w = shape
resized_h, resized_w = crop_meta["resized_size"]
left, top, right, bottom = crop_meta["crop_xyxy"]
mask_img = Image.fromarray(mask.astype(np.uint8) * 255)
resized = mask_img.resize((int(resized_w), int(resized_h)), Image.NEAREST)
cropped = resized.crop((int(left), int(top), int(right), int(bottom)))
arr = np.asarray(cropped, dtype=np.uint8) > 127
if arr.shape == (dst_h, dst_w):
return arr
out = np.zeros((dst_h, dst_w), dtype=bool)
out[: arr.shape[0], : arr.shape[1]] = arr
return out
def object_centric_resize_pad_image_and_mask(image, mask: np.ndarray, shape: tuple[int, int], *, margin_ratio: float, resample) -> tuple:
from PIL import Image
dst_h, dst_w = shape
src_w, src_h = image.size
x0, y0, x1, y1 = mask_bbox_xyxy(mask)
box_w = max(1, x1 - x0)
box_h = max(1, y1 - y0)
margin = max(1, int(round(max(box_w, box_h) * max(float(margin_ratio), 0.0))))
crop_x0 = max(0, x0 - margin)
crop_y0 = max(0, y0 - margin)
crop_x1 = min(src_w, x1 + margin)
crop_y1 = min(src_h, y1 + margin)
if crop_x1 <= crop_x0 or crop_y1 <= crop_y0:
processed_image, crop_meta = cover_resize_center_crop_image(image, shape, resample=resample)
processed_mask = cover_resize_center_crop_mask(mask, shape, crop_meta)
return processed_image, processed_mask, {"preprocess_mode": "camera_center_crop_empty_object", **crop_meta}
crop = image.crop((crop_x0, crop_y0, crop_x1, crop_y1))
crop_mask = mask[crop_y0:crop_y1, crop_x0:crop_x1]
crop_w, crop_h = crop.size
scale = min(dst_w / float(crop_w), dst_h / float(crop_h))
resized_w = max(1, min(dst_w, int(round(crop_w * scale))))
resized_h = max(1, min(dst_h, int(round(crop_h * scale))))
offset_x = (dst_w - resized_w) // 2
offset_y = (dst_h - resized_h) // 2
fill = local_background_color(crop, crop_mask)
canvas = Image.new("RGB", (dst_w, dst_h), fill)
resized = crop.resize((resized_w, resized_h), resample)
canvas.paste(resized, (offset_x, offset_y))
mask_canvas = np.zeros((dst_h, dst_w), dtype=bool)
resized_mask = Image.fromarray(crop_mask.astype(np.uint8) * 255).resize((resized_w, resized_h), Image.NEAREST)
mask_arr = np.asarray(resized_mask, dtype=np.uint8) > 127
mask_canvas[offset_y : offset_y + resized_h, offset_x : offset_x + resized_w] = mask_arr
return canvas, mask_canvas, {
"preprocess_mode": "object_centric",
"object_bbox_xyxy": [int(x0), int(y0), int(x1), int(y1)],
"source_crop_xyxy": [int(crop_x0), int(crop_y0), int(crop_x1), int(crop_y1)],
"resized_size": [int(resized_h), int(resized_w)],
"pad_xy": [int(offset_x), int(offset_y)],
"scale": float(scale),
"object_margin_ratio": float(margin_ratio),
}
def mask_bbox_xyxy(mask: np.ndarray) -> tuple[int, int, int, int]:
ys, xs = np.where(np.asarray(mask, dtype=bool))
if len(xs) == 0:
return (0, 0, int(mask.shape[1]), int(mask.shape[0]))
return (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1)
def local_background_color(image, mask: np.ndarray) -> tuple[int, int, int]:
arr = np.asarray(image.convert("RGB"), dtype=np.uint8)
bg = arr[~np.asarray(mask, dtype=bool)]
values = bg if bg.size else arr.reshape(-1, 3)
color = np.median(values.reshape(-1, 3), axis=0)
return tuple(int(np.clip(round(float(channel)), 0, 255)) for channel in color)
def remove_center_video_overlay(image_path: Path, object_mask: np.ndarray, output_dir: Path) -> tuple[Path, dict]:
from PIL import Image
image = Image.open(image_path).convert("RGB")
arr = np.asarray(image, dtype=np.uint8)
mask = np.asarray(object_mask, dtype=bool)
if mask.shape != arr.shape[:2]:
mask = resize_bool_mask(mask, arr.shape[:2])
overlay_mask = detect_center_video_overlay(arr, mask)
overlay_pixels = int(overlay_mask.sum())
metadata = {
"source_image": image_path.as_posix(),
"detected": overlay_pixels > 0,
"overlay_pixels": overlay_pixels,
}
if overlay_pixels == 0:
return image_path, metadata
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / "sam3d_overlay_removed.png"
try:
import cv2
inpaint_mask = (overlay_mask.astype(np.uint8) * 255)
cleaned = cv2.inpaint(arr, inpaint_mask, 5, cv2.INPAINT_TELEA)
except Exception:
cleaned = simple_mask_fill(arr, overlay_mask)
metadata["fallback"] = "simple_mask_fill"
Image.fromarray(cleaned.astype(np.uint8)).save(output_path)
metadata["image"] = output_path.as_posix()
return output_path, metadata
def detect_center_video_overlay(image: np.ndarray, object_mask: np.ndarray) -> np.ndarray:
height, width = image.shape[:2]
yy, xx = np.ogrid[:height, :width]
cx = width * 0.5
cy = height * 0.5
radius = max(8.0, min(height, width) * 0.11)
disk = (xx - cx) ** 2 + (yy - cy) ** 2 <= radius ** 2
rgb = image.astype(np.float32) / 255.0
max_channel = rgb.max(axis=2)
min_channel = rgb.min(axis=2)
saturation = (max_channel - min_channel) / np.maximum(max_channel, 1e-6)
bright_neutral = (max_channel > 0.72) & (saturation < 0.28)
raw = disk & object_mask & bright_neutral
if int(raw.sum()) < max(20, int(radius)):
return np.zeros((height, width), dtype=bool)
return dilate_bool_mask(raw, radius=3)
def resize_bool_mask(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray:
from PIL import Image
height, width = shape
resized = Image.fromarray(mask.astype(np.uint8) * 255).resize((width, height), Image.NEAREST)
return np.asarray(resized, dtype=np.uint8) > 127
def dilate_bool_mask(mask: np.ndarray, *, radius: int) -> np.ndarray:
try:
import cv2
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1))
return cv2.dilate(mask.astype(np.uint8), kernel, iterations=1) > 0
except Exception:
padded = np.pad(mask, radius, mode="constant", constant_values=False)
out = np.zeros_like(mask, dtype=bool)
for dy in range(radius * 2 + 1):
for dx in range(radius * 2 + 1):
if (dx - radius) ** 2 + (dy - radius) ** 2 <= radius ** 2:
out |= padded[dy : dy + mask.shape[0], dx : dx + mask.shape[1]]
return out
def simple_mask_fill(image: np.ndarray, mask: np.ndarray) -> np.ndarray:
cleaned = image.copy()
if not mask.any():
return cleaned
yy, xx = np.where(mask)
y0, y1 = max(0, int(yy.min()) - 5), min(image.shape[0], int(yy.max()) + 6)
x0, x1 = max(0, int(xx.min()) - 5), min(image.shape[1], int(xx.max()) + 6)
patch = image[y0:y1, x0:x1]
patch_mask = mask[y0:y1, x0:x1]
fill_values = patch[~patch_mask]
fill = np.median(fill_values, axis=0) if fill_values.size else np.median(image.reshape(-1, 3), axis=0)
cleaned[mask] = fill.astype(np.uint8)
return cleaned
PLY_NUMPY_DTYPES = {
"char": "i1",
"int8": "i1",
"uchar": "u1",
"uint8": "u1",
"short": "i2",
"int16": "i2",
"ushort": "u2",
"uint16": "u2",
"int": "i4",
"int32": "i4",
"uint": "u4",
"uint32": "u4",
"float": "f4",
"float32": "f4",
"double": "f8",
"float64": "f8",
}
def canonicalize_gaussian_to_mesh_frame(gaussian_ply: Path, mesh_obj: Path) -> dict:
table = load_ply_vertex_table(gaussian_ply)
mesh_vertices = load_obj_vertices(mesh_obj)
if table.array.size == 0 or mesh_vertices.size == 0:
return {
"enabled": True,
"status": "skipped_empty_geometry",
"gaussian": gaussian_ply.as_posix(),
"mesh": mesh_obj.as_posix(),
}
xyz_indices = indices_for(table.property_names, ("x", "y", "z"))
points = table.array[:, xyz_indices]
gaussian_extent = extent(points)
mesh_extent = extent(mesh_vertices)
rotation, mapping, signs = canonical_axis_rotation(gaussian_extent, mesh_extent)
changed = not np.allclose(rotation, np.eye(3), atol=1e-6)
if changed:
apply_axis_rotation_to_ply_table(table, rotation)
write_ply_vertex_table(gaussian_ply, table)
return {
"enabled": True,
"status": "ok",
"changed": bool(changed),
"gaussian": gaussian_ply.as_posix(),
"mesh": mesh_obj.as_posix(),
"gaussian_extent_before": vector_json(gaussian_extent),
"mesh_extent": vector_json(mesh_extent),
"axis_mapping": {
"source": "extent_permutation",
"gaussian_axes_to_mesh_axes": [int(value) for value in mapping],
"mesh_axis_signs": [int(value) for value in signs],
},
"rotation_matrix": [[round(float(value), 7) for value in row] for row in rotation.tolist()],
}
def canonical_axis_rotation(gaussian_extent: np.ndarray, mesh_extent: np.ndarray) -> tuple[np.ndarray, list[int], list[int]]:
gaussian_order = list(np.argsort(-np.asarray(gaussian_extent, dtype=np.float32)))
mesh_order = list(np.argsort(-np.asarray(mesh_extent, dtype=np.float32)))
mapping = [0, 1, 2]
for mesh_axis, gaussian_axis in zip(mesh_order, gaussian_order):
mapping[int(mesh_axis)] = int(gaussian_axis)
signs = [1, 1, 1]
rotation = rotation_from_axis_mapping(mapping, signs)
if np.linalg.det(rotation) < 0.0:
signs[int(mesh_order[-1])] = -1
rotation = rotation_from_axis_mapping(mapping, signs)
return rotation.astype(np.float32), mapping, signs
def rotation_from_axis_mapping(mapping: list[int], signs: list[int]) -> np.ndarray:
rotation = np.zeros((3, 3), dtype=np.float32)
for mesh_axis, gaussian_axis in enumerate(mapping):
rotation[mesh_axis, gaussian_axis] = float(signs[mesh_axis])
return rotation
def apply_axis_rotation_to_ply_table(table: PlyVertexTable, rotation: np.ndarray) -> None:
names = table.property_names
xyz_indices = indices_for(names, ("x", "y", "z"))
table.array[:, xyz_indices] = table.array[:, xyz_indices] @ rotation.T
if all(name in names for name in ("nx", "ny", "nz")):
normal_indices = indices_for(names, ("nx", "ny", "nz"))
table.array[:, normal_indices] = table.array[:, normal_indices] @ rotation.T
if all(name in names for name in ("rot_0", "rot_1", "rot_2", "rot_3")):
rot_indices = indices_for(names, ("rot_0", "rot_1", "rot_2", "rot_3"))
transform_quat = quat_wxyz_from_rotation_matrix(rotation)
for idx in range(table.array.shape[0]):
table.array[idx, rot_indices] = quat_multiply(transform_quat, normalize_quat(table.array[idx, rot_indices]))
def load_ply_vertex_table(path: Path) -> PlyVertexTable:
data = path.read_bytes()
header_marker = b"end_header"
marker_index = data.find(header_marker)
if marker_index < 0:
raise SystemExit(f"PLY missing end_header: {path}")
body_start = data.find(b"\n", marker_index)
if body_start < 0:
raise SystemExit(f"PLY header missing final newline: {path}")
body_start += 1
header_lines = data[:body_start].decode("ascii", errors="ignore").splitlines()
fmt = "ascii"
vertex_count = 0
property_names: list[str] = []
property_types: list[str] = []
current_element = ""
for line in header_lines:
parts = line.strip().split()
if len(parts) >= 3 and parts[0] == "format":
fmt = parts[1]
elif len(parts) >= 3 and parts[0] == "element":
current_element = parts[1]
if current_element == "vertex":
vertex_count = int(parts[2])
elif len(parts) >= 3 and parts[0] == "property" and current_element == "vertex":
if parts[1] == "list":
raise SystemExit(f"PLY vertex list properties are not supported for canonicalization: {path}")
property_types.append(parts[1])
property_names.append(parts[-1])
if vertex_count <= 0:
return PlyVertexTable(
header_lines=header_lines,
body_start=body_start,
fmt=fmt,
vertex_count=0,
property_names=property_names,
property_types=property_types,
array=np.zeros((0, len(property_names)), dtype=np.float32),
)
if not all(name in property_names for name in ("x", "y", "z")):
raise SystemExit(f"PLY must expose x/y/z vertex properties: {path}")
body = data[body_start:]
if fmt == "ascii":
array = load_ascii_ply_vertex_array(body, vertex_count, len(property_names))
elif fmt in {"binary_little_endian", "binary_big_endian"}:
array = load_binary_ply_vertex_array(body, vertex_count, property_types, little_endian=fmt == "binary_little_endian")
else:
raise SystemExit(f"unsupported PLY format for canonicalization: {fmt}")
return PlyVertexTable(
header_lines=header_lines,
body_start=body_start,
fmt=fmt,
vertex_count=vertex_count,
property_names=property_names,
property_types=property_types,
array=array.astype(np.float32, copy=False),
)
def load_ascii_ply_vertex_array(body: bytes, vertex_count: int, property_count: int) -> np.ndarray:
rows = []
for line in body.decode("ascii", errors="ignore").splitlines():
if len(rows) >= vertex_count:
break
stripped = line.strip()
if not stripped:
continue
values = [float(value) for value in stripped.split()[:property_count]]
if len(values) == property_count:
rows.append(values)
return np.asarray(rows, dtype=np.float32)
def load_binary_ply_vertex_array(body: bytes, vertex_count: int, property_types: list[str], *, little_endian: bool) -> np.ndarray:
byte_order = "<" if little_endian else ">"
try:
dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(property_types)])
except KeyError as exc:
raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc
structured = np.frombuffer(body, dtype=dtype, count=vertex_count)
columns = [structured[f"f{idx}"].astype(np.float32) for idx in range(len(property_types))]
return np.stack(columns, axis=1) if columns else np.zeros((vertex_count, 0), dtype=np.float32)
def write_ply_vertex_table(path: Path, table: PlyVertexTable) -> None:
header = "\n".join(table.header_lines) + "\n"
if table.fmt == "ascii":
lines = [" ".join(f"{float(value):.9g}" for value in row) for row in table.array]
path.write_text(header + "\n".join(lines) + "\n", encoding="ascii")
return
if table.fmt in {"binary_little_endian", "binary_big_endian"}:
byte_order = "<" if table.fmt == "binary_little_endian" else ">"
try:
dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(table.property_types)])
except KeyError as exc:
raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc
structured = np.empty(table.array.shape[0], dtype=dtype)
for idx in range(table.array.shape[1]):
structured[f"f{idx}"] = table.array[:, idx].astype(dtype[idx])
path.write_bytes(header.encode("ascii") + structured.tobytes())
return
raise SystemExit(f"unsupported PLY format for canonicalization: {table.fmt}")
def load_obj_vertices(path: Path) -> np.ndarray:
vertices = []
with path.open("r", encoding="utf-8", errors="ignore") as f:
for line in f:
if not line.startswith("v "):
continue
parts = line.split()
if len(parts) >= 4:
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
return np.asarray(vertices, dtype=np.float32)
def indices_for(names: list[str], required: tuple[str, ...]) -> list[int]:
return [names.index(name) for name in required]
def extent(points: np.ndarray) -> np.ndarray:
if points.size == 0:
return np.zeros(3, dtype=np.float32)
return np.asarray(points.max(axis=0) - points.min(axis=0), dtype=np.float32)
def normalize_quat(raw: np.ndarray) -> np.ndarray:
quat = np.asarray(raw, dtype=np.float32).reshape(4)
norm = float(np.linalg.norm(quat))
if norm <= 1e-8:
return np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32)
return quat / norm
def quat_multiply(a: np.ndarray, b: np.ndarray) -> np.ndarray:
aw, ax, ay, az = normalize_quat(a)
bw, bx, by, bz = normalize_quat(b)
return normalize_quat(
np.asarray(
[
aw * bw - ax * bx - ay * by - az * bz,
aw * bx + ax * bw + ay * bz - az * by,
aw * by - ax * bz + ay * bw + az * bx,
aw * bz + ax * by - ay * bx + az * bw,
],
dtype=np.float32,
)
)
def quat_wxyz_from_rotation_matrix(rotation: np.ndarray) -> np.ndarray:
matrix = np.asarray(rotation, dtype=np.float64).reshape(3, 3)
trace = float(np.trace(matrix))
if trace > 0.0:
s = np.sqrt(trace + 1.0) * 2.0
quat = np.asarray(
[
0.25 * s,
(matrix[2, 1] - matrix[1, 2]) / s,
(matrix[0, 2] - matrix[2, 0]) / s,
(matrix[1, 0] - matrix[0, 1]) / s,
],
dtype=np.float64,
)
else:
axis = int(np.argmax(np.diag(matrix)))
if axis == 0:
s = np.sqrt(1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2]) * 2.0
quat = np.asarray(
[
(matrix[2, 1] - matrix[1, 2]) / s,
0.25 * s,
(matrix[0, 1] + matrix[1, 0]) / s,
(matrix[0, 2] + matrix[2, 0]) / s,
],
dtype=np.float64,
)
elif axis == 1:
s = np.sqrt(1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2]) * 2.0
quat = np.asarray(
[
(matrix[0, 2] - matrix[2, 0]) / s,
(matrix[0, 1] + matrix[1, 0]) / s,
0.25 * s,
(matrix[1, 2] + matrix[2, 1]) / s,
],
dtype=np.float64,
)
else:
s = np.sqrt(1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1]) * 2.0
quat = np.asarray(
[
(matrix[1, 0] - matrix[0, 1]) / s,
(matrix[0, 2] + matrix[2, 0]) / s,
(matrix[1, 2] + matrix[2, 1]) / s,
0.25 * s,
],
dtype=np.float64,
)
return normalize_quat(quat.astype(np.float32))
def vector_json(values: np.ndarray) -> list[float]:
return [round(float(value), 7) for value in np.asarray(values, dtype=np.float32).reshape(-1).tolist()]
def pose_from_layout(layout: dict) -> Pose:
position = layout.get("position", layout.get("translation", [0.0, 0.0, 0.0]))
scale = layout.get("scale", [1.0, 1.0, 1.0])
quat = layout.get("quat_wxyz", layout.get("quaternion_wxyz", layout.get("rotation", [1.0, 0.0, 0.0, 0.0])))
if "quaternion_xyzw" in layout and "quat_wxyz" not in layout and "quaternion_wxyz" not in layout and "rotation" not in layout:
qx, qy, qz, qw = vector_from_value(layout["quaternion_xyzw"], 4, "quaternion_xyzw")
quat = (qw, qx, qy, qz)
return Pose(
position=vector_from_value(position, 3, "position"),
quat_wxyz=vector_from_value(quat, 4, "quat_wxyz"),
scale=vector_from_value(scale, 3, "scale", broadcast_scalar=True),
)
def run_sam3d_inference(
inference,
image: np.ndarray,
mask_np: np.ndarray,
*,
seed: int,
texture_baking: bool,
texture_render_backend: str = "gsplat",
texture_baking_engine: str = "pytorch3d",
) -> dict:
if not texture_baking:
return inference(image, mask_np, seed=seed)
patch_texture_baking_render_backend(texture_render_backend)
pipeline = getattr(inference, "_pipeline", None)
merge_mask_to_rgba = getattr(inference, "merge_mask_to_rgba", None)
if pipeline is None or not hasattr(pipeline, "run") or merge_mask_to_rgba is None:
raise SystemExit("--texture-baking requires SAM-3D notebook Inference._pipeline.run support")
if hasattr(pipeline, "rendering_engine"):
pipeline.rendering_engine = texture_baking_engine
rgba = merge_mask_to_rgba(image, mask_np)
options = {
"stage1_only": False,
"with_mesh_postprocess": False,
"with_texture_baking": True,
"with_layout_postprocess": False,
"use_vertex_color": False,
"stage1_inference_steps": None,
"pointmap": None,
}
return pipeline.run(rgba, None, seed, **supported_kwargs(pipeline.run, options))
def patch_texture_baking_render_backend(backend: str) -> bool:
if backend in {"", "auto", "inria"}:
return False
if backend != "gsplat":
raise SystemExit(f"unsupported SAM-3D texture render backend: {backend}")
render_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.render_utils")
postprocessing_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.postprocessing_utils")
def render_multiview_with_backend(sample, resolution=512, nviews=30):
radius = 2
fov = 40
cameras = [render_utils.sphere_hammersley_sequence(index, nviews) for index in range(nviews)]
yaws = [camera[0] for camera in cameras]
pitches = [camera[1] for camera in cameras]
extrinsics, intrinsics = render_utils.yaw_pitch_r_fov_to_extrinsics_intrinsics(yaws, pitches, radius, fov)
result = render_utils.render_frames(
sample,
extrinsics,
intrinsics,
{"resolution": resolution, "bg_color": (0, 0, 0), "backend": "gsplat"},
)
return result["color"], extrinsics, intrinsics
postprocessing_utils.render_multiview = render_multiview_with_backend
return True
def supported_kwargs(callable_obj, options: dict[str, object]) -> dict[str, object]:
signature = inspect.signature(callable_obj)
if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()):
return options
return {key: value for key, value in options.items() if key in signature.parameters}
def save_gaussian(output: dict, path: Path) -> None:
for key in ("gs", "gaussian", "gaussians"):
if key not in output:
continue
value = first_value(output[key])
if hasattr(value, "save_ply"):
path.parent.mkdir(parents=True, exist_ok=True)
value.save_ply(path.as_posix())
return
raise SystemExit(f"SAM 3D output does not expose a Gaussian save_ply method; keys={sorted(output.keys())}")
def export_mesh(output: dict, path: Path, *, prefer_textured: bool = False) -> bool:
keys = ("glb", "mesh", "trimesh", "obj") if prefer_textured else ("mesh", "trimesh", "glb", "obj")
for key in keys:
if key not in output:
continue
value = first_value(output[key])
if hasattr(value, "export"):
path.parent.mkdir(parents=True, exist_ok=True)
value.export(path.as_posix())
return True
if isinstance(value, (str, os.PathLike)):
source = Path(value)
if source.is_file():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(source.read_bytes())
return True
return False
def first_value(value):
if isinstance(value, (list, tuple)):
return value[0] if value else value
return value
def vector_from_value(value, size: int, field_name: str, *, broadcast_scalar: bool = False) -> tuple[float, ...]:
if hasattr(value, "detach"):
value = value.detach().cpu().numpy()
arr = np.asarray(value, dtype=np.float64).squeeze()
if arr.ndim == 0:
if broadcast_scalar:
return tuple(round(float(arr), 7) for _ in range(size))
if size == 1:
return (round(float(arr), 7),)
flat = arr.reshape(-1)
if flat.size != size:
raise SystemExit(f"{field_name} must have {size} values, got shape {arr.shape}")
return tuple(round(float(x), 7) for x in flat)
def write_bbox_mesh_from_mask(path: Path, mask) -> None:
import numpy as np
ys, xs = np.where(mask)
if ys.size == 0:
extent = np.array([0.1, 0.1, 0.1], dtype=np.float32)
else:
extent = np.array([max((xs.max() - xs.min()) / 512.0, 0.05), 0.08, max((ys.max() - ys.min()) / 512.0, 0.05)], dtype=np.float32)
half = extent / 2.0
vertices = [
(-half[0], -half[1], -half[2]),
(half[0], -half[1], -half[2]),
(half[0], half[1], -half[2]),
(-half[0], half[1], -half[2]),
(-half[0], -half[1], half[2]),
(half[0], -half[1], half[2]),
(half[0], half[1], half[2]),
(-half[0], half[1], half[2]),
]
faces = [(1, 2, 3), (1, 3, 4), (5, 8, 7), (5, 7, 6), (1, 5, 6), (1, 6, 2), (2, 6, 7), (2, 7, 3), (3, 7, 8), (3, 8, 4), (4, 8, 5), (4, 5, 1)]
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="ascii") as f:
f.write("o sam3d_collision_fallback\n")
for v in vertices:
f.write(f"v {v[0]:.7g} {v[1]:.7g} {v[2]:.7g}\n")
for face in faces:
f.write(f"f {face[0]} {face[1]} {face[2]}\n")
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,296 @@
#!/usr/bin/env python3
"""Speedy-Splat pruning adapter for Real2Sim 3DGS assets.
This adapter bridges Real2Sim's exported PLY assets into the official
Speedy-Splat pruning implementation. It loads the input PLY with
Speedy-Splat's GaussianModel, builds a camera from the AnySplat camera JSON,
calls the official train.prune function, and writes the pruned PLY.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
import traceback
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import numpy as np
def main() -> int:
parser = argparse.ArgumentParser(description="Prune a 3DGS PLY with official Speedy-Splat scoring")
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--image", type=Path, required=True, help="Source image used to build the pruning camera")
parser.add_argument("--cameras", type=Path, required=True, help="AnySplat cameras.json artifact")
parser.add_argument("--keep-ratio", type=float, default=float(os.environ.get("REAL2SIM_SPEEDY_KEEP_RATIO", "0.85")))
parser.add_argument("--keep-count", type=int, default=None)
parser.add_argument("--speedy-root", type=Path, default=os.environ.get("REAL2SIM_SPEEDY_SPLAT_ROOT"))
parser.add_argument("--background", choices=("black", "white"), default=os.environ.get("REAL2SIM_SPEEDY_BACKGROUND", "black"))
parser.add_argument("--metadata", type=Path, default=None)
args = parser.parse_args()
if args.speedy_root is None:
raise SystemExit("Speedy-Splat root is required via --speedy-root or REAL2SIM_SPEEDY_SPLAT_ROOT")
if not args.input.is_file():
raise SystemExit(f"input PLY not found: {args.input}")
if not args.image.is_file():
raise SystemExit(f"source image not found: {args.image}")
if not args.cameras.is_file():
raise SystemExit(f"camera JSON not found: {args.cameras}")
sys.path.insert(0, args.speedy_root.as_posix())
import torch
from PIL import Image
from plyfile import PlyData
from scene.cameras import Camera
from scene.gaussian_model import GaussianModel
from train import prune as speedy_prune
props = [prop.name for prop in PlyData.read(args.input)["vertex"].properties]
sh_degree = infer_sh_degree(props)
gaussians = GaussianModel(sh_degree)
gaussians.load_ply(args.input.as_posix())
gaussians.spatial_lr_scale = 1.0
gaussians.training_setup(speedy_training_args())
gaussians.max_radii2D = torch.zeros((gaussians.get_xyz.shape[0],), device="cuda")
prune_policy = install_exact_count_pruning(torch, gaussians)
camera_payload = load_json(args.cameras)
intrinsics = require_intrinsics(camera_payload)
width = int(intrinsics["width"])
height = int(intrinsics["height"])
image_tensor = load_image_tensor(Image, torch, args.image, width, height)
view_index = image_view_index(camera_payload, args.image)
extrinsic = first_matrix(camera_payload.get("pred_context_pose", {}).get("extrinsic"), rows=4, cols=4, view_index=view_index)
rotation, translation = speedy_camera_rt(extrinsic)
camera = Camera(
colmap_id=0,
R=rotation,
T=translation,
FoVx=focal_to_fov(float(intrinsics["fx"]), width),
FoVy=focal_to_fov(float(intrinsics["fy"]), height),
image=image_tensor,
gt_alpha_mask=None,
image_name=args.image.stem,
uid=0,
)
prune_ratio = prune_ratio_from_keep(args.keep_ratio, args.keep_count, int(gaussians.get_xyz.shape[0]))
background = torch.tensor([1.0, 1.0, 1.0] if args.background == "white" else [0.0, 0.0, 0.0], dtype=torch.float32, device="cuda")
scene = SingleCameraScene([camera])
pipe = SimpleNamespace(debug=False, convert_SHs_python=False, compute_cov3D_python=False)
prune_metrics = run_speedy_prune(torch, speedy_prune, scene, gaussians, pipe, background, prune_ratio)
args.output.parent.mkdir(parents=True, exist_ok=True)
gaussians.save_ply(args.output.as_posix())
metadata_path = args.metadata or args.output.with_suffix(".speedy_splat.json")
write_json(
metadata_path,
{
"backend": "speedy_splat",
"speedy_root": args.speedy_root.as_posix(),
"input": args.input.as_posix(),
"output": args.output.as_posix(),
"image": args.image.as_posix(),
"cameras": args.cameras.as_posix(),
"sh_degree": sh_degree,
"keep_ratio": args.keep_ratio,
"keep_count": args.keep_count,
"camera_view_index": view_index,
"prune_ratio": prune_ratio,
"input_gaussians": int(len(PlyData.read(args.input)["vertex"])),
"output_gaussians": int(gaussians.get_xyz.shape[0]),
"prune_metrics": prune_metrics,
"prune_policy": prune_policy,
},
)
return 0
class SingleCameraScene:
def __init__(self, cameras: list[Any]) -> None:
self._cameras = cameras
def getTrainCameras(self) -> list[Any]:
return self._cameras
def run_speedy_prune(
torch_module: Any,
speedy_prune: Any,
scene: Any,
gaussians: Any,
pipe: Any,
background: Any,
prune_ratio: float,
) -> dict[str, Any]:
try:
metrics = speedy_prune(scene, gaussians, pipe, background, prune_ratio)
synchronize_cuda(torch_module)
return dict(metrics) if isinstance(metrics, dict) else {"status": "ok"}
except RuntimeError as exc:
trace = traceback.format_exc()
if not is_cuda_event_timing_failure(exc, trace):
raise
synchronize_cuda(torch_module)
return {
"status": "ok_with_cuda_timing_warning",
"timing_warning": first_line(str(exc)),
}
def install_exact_count_pruning(torch_module: Any, gaussians: Any) -> str:
def prune_gaussians(percent: float, import_score: Any) -> None:
prune_gaussians_exact_count(torch_module, gaussians, percent, import_score)
gaussians.prune_gaussians = prune_gaussians
return "speedy_splat_exact_count_tie_safe"
def prune_gaussians_exact_count(torch_module: Any, gaussians: Any, percent: float, import_score: Any) -> None:
scores = import_score.reshape(-1)
total = int(scores.shape[0])
prune_count = exact_prune_count(total, percent)
prune_mask = torch_module.zeros((total,), dtype=torch_module.bool, device=scores.device)
if prune_count > 0:
try:
order = torch_module.argsort(scores, stable=True)
except TypeError:
order = torch_module.argsort(scores)
prune_mask[order[:prune_count]] = True
gaussians.prune_points(prune_mask)
def exact_prune_count(total: int, percent: float) -> int:
if total <= 1:
return 0
if percent <= 0.0:
return 0
return min(max(int(round(total * percent)), 0), total - 1)
def is_cuda_event_timing_failure(exc: RuntimeError, trace: str) -> bool:
message = str(exc).lower()
return "device not ready" in message and "elapsed_time" in trace
def synchronize_cuda(torch_module: Any) -> None:
cuda = getattr(torch_module, "cuda", None)
synchronize = getattr(cuda, "synchronize", None)
if callable(synchronize):
synchronize()
def first_line(value: str) -> str:
return value.splitlines()[0] if value.splitlines() else value
def speedy_training_args() -> SimpleNamespace:
return SimpleNamespace(
percent_dense=0.01,
position_lr_init=0.00016,
position_lr_final=0.0000016,
position_lr_delay_mult=0.01,
position_lr_max_steps=30_000,
feature_lr=0.0025,
opacity_lr=0.05,
scaling_lr=0.005,
rotation_lr=0.001,
)
def infer_sh_degree(property_names: list[str]) -> int:
f_rest = [name for name in property_names if name.startswith("f_rest_")]
coeffs = (len(f_rest) + 3) / 3
degree = math.sqrt(coeffs) - 1
rounded = int(round(degree))
if rounded < 0 or not math.isclose(degree, rounded, rel_tol=0.0, abs_tol=1e-6):
raise SystemExit(f"cannot infer spherical harmonics degree from {len(f_rest)} f_rest properties")
return rounded
def prune_ratio_from_keep(keep_ratio: float, keep_count: int | None, total: int) -> float:
if total <= 0:
raise SystemExit("input PLY has no gaussians")
if keep_count is not None:
if keep_count <= 0:
raise SystemExit("--keep-count must be positive")
keep_ratio = min(1.0, max(0.0, keep_count / total))
if not 0.0 < keep_ratio <= 1.0:
raise SystemExit("--keep-ratio must be in (0, 1]")
return max(0.0, 1.0 - keep_ratio)
def require_intrinsics(payload: dict[str, Any]) -> dict[str, float]:
intrinsics = payload.get("intrinsics")
required = ("width", "height", "fx", "fy")
if not isinstance(intrinsics, dict) or any(key not in intrinsics for key in required):
raise SystemExit(f"camera JSON must contain intrinsics with {', '.join(required)}")
return intrinsics
def focal_to_fov(focal: float, pixels: int) -> float:
if focal <= 0:
raise SystemExit("camera focal length must be positive")
return 2.0 * math.atan(float(pixels) / (2.0 * focal))
def image_view_index(payload: dict[str, Any], image_path: Path) -> int:
image_paths = payload.get("image_paths")
if not isinstance(image_paths, list):
return 0
target = image_path.expanduser().resolve()
for index, raw_path in enumerate(image_paths):
candidate = Path(str(raw_path)).expanduser().resolve()
if candidate == target:
return index
return 0
def first_matrix(value: Any, *, rows: int, cols: int, view_index: int = 0) -> np.ndarray:
arr = np.asarray(value, dtype=np.float32)
while arr.ndim > 2 and arr.shape[0] == 1:
arr = arr[0]
if arr.ndim == 3 and arr.shape[1:] == (rows, cols):
index = min(max(int(view_index), 0), arr.shape[0] - 1)
arr = arr[index]
if arr.shape != (rows, cols):
raise SystemExit(f"expected matrix shape {(rows, cols)}, got {arr.shape}")
return arr
def speedy_camera_rt(extrinsic: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
# Speedy-Splat's Camera builds world_view_transform with R.T and T.
return np.asarray(extrinsic[:3, :3], dtype=np.float32).T, np.asarray(extrinsic[:3, 3], dtype=np.float32)
def load_image_tensor(image_module: Any, torch_module: Any, path: Path, width: int, height: int):
image = image_module.open(path).convert("RGB").resize((width, height))
arr = np.asarray(image, dtype=np.float32) / 255.0
return torch_module.from_numpy(arr).permute(2, 0, 1).contiguous()
def load_json(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
if not isinstance(data, dict):
raise SystemExit(f"expected JSON object: {path}")
return data
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
f.write("\n")
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,299 @@
#!/usr/bin/env python3
"""Expand one RGB reference image into synthetic multi-view frames.
This is an optional Real2Sim adapter. It calls an OpenAI-style image edit API
with one reference image, saves the returned contact sheet, and slices it into
ordered view images that AnySplat can consume through ``--frames-dir``.
"""
from __future__ import annotations
import argparse
import base64
import json
import os
from pathlib import Path
from typing import Any
VIEW_NAMES = [
"01_front_main",
"02_front_left_45",
"03_left_90",
"04_rear_left_135",
"05_rear_back",
"06_rear_right_225",
"07_right_270",
"08_top_oblique",
]
def main() -> int:
parser = argparse.ArgumentParser(description="Generate synthetic multi-view frames from one image")
parser.add_argument("--image", type=Path, required=True, help="Reference RGB image")
parser.add_argument("--output-dir", type=Path, required=True, help="Directory for generated metadata and contact sheet")
parser.add_argument("--frames-dir", type=Path, required=True, help="Directory where sliced views are written")
parser.add_argument("--prompt", default="", help="Image edit prompt. A real2sim prompt is built when omitted.")
parser.add_argument("--mode", choices=["object", "scene"], default="object", help="Default prompt mode when --prompt is omitted")
parser.add_argument("--api-url", default=os.environ.get("REAL2SIM_IMAGE_EDIT_API_URL", "https://chatapi.applyma.com/v1/images/edits"))
parser.add_argument("--api-key-env", default="REAL2SIM_IMAGE_EDIT_API_KEY", help="Environment variable that contains the API key")
parser.add_argument("--model", default=os.environ.get("REAL2SIM_IMAGE_EDIT_MODEL", "gpt-image-2"))
parser.add_argument("--size", default=os.environ.get("REAL2SIM_IMAGE_EDIT_SIZE", "1536x1024"))
parser.add_argument("--quality", default=os.environ.get("REAL2SIM_IMAGE_EDIT_QUALITY", "auto"))
parser.add_argument("--response-format", default="b64_json")
parser.add_argument("--cols", type=int, default=4)
parser.add_argument("--rows", type=int, default=2)
parser.add_argument(
"--cell-crop-ratio",
type=float,
default=float(os.environ.get("REAL2SIM_IMAGE_EDIT_CELL_CROP_RATIO", "0.015")),
help="Small per-cell edge crop used to remove generated contact-sheet gutters",
)
parser.add_argument(
"--keep-label-bars",
action="store_true",
help="Keep generated panel label bars instead of automatically cropping dark bottom captions",
)
parser.add_argument("--timeout", type=float, default=float(os.environ.get("REAL2SIM_IMAGE_EDIT_TIMEOUT", "300")))
args = parser.parse_args()
if not args.image.is_file():
raise SystemExit(f"reference image not found: {args.image}")
api_key = os.environ.get(args.api_key_env)
if not api_key:
raise SystemExit(f"missing API key: set {args.api_key_env}")
args.output_dir.mkdir(parents=True, exist_ok=True)
prompt = args.prompt.strip() or build_real2sim_multiview_prompt("the reference object", mode=args.mode)
(args.output_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
payload = request_image_edit(
api_url=args.api_url,
api_key=api_key,
image=args.image,
prompt=prompt,
model=args.model,
size=args.size,
quality=args.quality,
response_format=args.response_format,
timeout=args.timeout,
)
write_response_metadata(payload, args.output_dir / "response_meta.json")
contact_sheet = args.output_dir / "contact_sheet.png"
save_first_image_payload(payload, contact_sheet, timeout=args.timeout)
views = split_contact_sheet(
contact_sheet,
args.frames_dir,
cols=args.cols,
rows=args.rows,
cell_crop_ratio=args.cell_crop_ratio,
remove_label_bars=not args.keep_label_bars,
)
manifest = {
"backend": "synthetic_multiview_expand",
"reference_image": args.image.as_posix(),
"contact_sheet": contact_sheet.as_posix(),
"frames_dir": args.frames_dir.as_posix(),
"views": [path.as_posix() for path in views],
"grid": {"cols": args.cols, "rows": args.rows},
"cell_crop_ratio": args.cell_crop_ratio,
"model": args.model,
"size": args.size,
"quality": args.quality,
}
(args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
return 0
def build_real2sim_multiview_prompt(subject: str, *, mode: str = "object") -> str:
if mode == "scene":
return (
"Create a single photorealistic real2sim scene-completion multiview contact sheet from the reference image. "
f"Scene identity to preserve: {subject}. Treat the reference as the canonical source view and preserve the exact visible layout, "
"floor plane, walls, doors, windows, static furniture, fixtures, material colors, lighting direction, and object identities. "
"Output exactly 8 equal portrait-photo panels in a clean 4 columns x 2 rows grid, filling one 1536x1024 image. "
"Use thin white gutters between panels. No panel titles, labels, captions, numbers, UI icons, or watermarks. "
"Views, left to right, top row then bottom row: source/front view, front-left 45 degree view, left side 90 degree view, "
"rear-left 135 degree view, rear/back view, rear-right 225 degree view, right side 270 degree view, and top-oblique "
"30 to 45 degree view. Infer unseen regions conservatively as physically plausible continuations of the visible scene, "
"keeping camera height, focal length, scale, horizon, navigable ground continuity, occlusion relationships, and shadows consistent. "
"Do not turn the scene into an isolated object turntable, do not invent a different room, and do not add people."
)
return (
"Create a single photorealistic real2sim multiview capture contact sheet from the reference image. "
f"Subject identity to preserve: {subject}. Keep the same object geometry, material, colors, labels, markings, and scale. "
"Output exactly 8 equal portrait-photo panels in a clean 4 columns x 2 rows grid, filling one 1536x1024 image. "
"Use thin white gutters between panels. No panel titles, labels, captions, numbers, UI icons, or watermarks. "
"Views, left to right, top row then bottom row: front main view, front-left 45 degree view, left side 90 degree view, "
"rear-left 135 degree view, rear/back view, rear-right 225 degree view, right side 270 degree view, and top-oblique "
"30 to 45 degree view. Keep the object stationary on the same tabletop, consistent focal length, consistent soft lighting, "
"sharp focus, no motion blur, no hands, no people blocking the object, no play button overlays. Keep the object fully inside "
"every panel with 10 to 15 percent silhouette margin and preserve reconstruction-useful texture and contour details."
)
def request_image_edit(
*,
api_url: str,
api_key: str,
image: Path,
prompt: str,
model: str,
size: str,
quality: str,
response_format: str,
timeout: float,
) -> dict[str, Any]:
try:
import requests
except Exception as exc:
raise SystemExit(f"synthetic multiview expansion requires requests: {exc}") from exc
with image.open("rb") as image_file:
response = requests.post(
api_url,
headers={"Authorization": f"Bearer {api_key}"},
data={
"model": model,
"prompt": prompt,
"size": size,
"quality": quality,
"response_format": response_format,
"n": "1",
},
files=[("image", (image.name, image_file, mime_type_for_image(image)))],
timeout=timeout,
)
if response.status_code >= 400:
raise SystemExit(f"image edit API failed with status {response.status_code}: {response.text[:1000]}")
try:
payload = response.json()
except Exception as exc:
raise SystemExit(f"image edit API returned non-JSON response: {exc}") from exc
if not isinstance(payload, dict):
raise SystemExit("image edit API response must be a JSON object")
return payload
def write_response_metadata(payload: dict[str, Any], path: Path) -> None:
redacted = {key: value for key, value in payload.items() if key != "data"}
if isinstance(payload.get("data"), list):
redacted["data"] = [
{key: value for key, value in item.items() if key != "b64_json"}
for item in payload["data"]
if isinstance(item, dict)
]
path.write_text(json.dumps(redacted, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def save_first_image_payload(payload: dict[str, Any], output: Path, *, timeout: float) -> None:
data = payload.get("data")
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
raise SystemExit("image edit API response missing data[0]")
item = data[0]
if item.get("b64_json"):
output.write_bytes(base64.b64decode(str(item["b64_json"])))
return
if item.get("url"):
try:
import requests
except Exception as exc:
raise SystemExit(f"downloading URL response requires requests: {exc}") from exc
response = requests.get(str(item["url"]), timeout=timeout)
if response.status_code >= 400:
raise SystemExit(f"failed to download generated image: HTTP {response.status_code}")
output.write_bytes(response.content)
return
raise SystemExit("image edit API response data[0] must contain b64_json or url")
def split_contact_sheet(
contact_sheet: Path,
frames_dir: Path,
*,
cols: int = 4,
rows: int = 2,
cell_crop_ratio: float = 0.015,
remove_label_bars: bool = True,
) -> list[Path]:
from PIL import Image
if cols <= 0 or rows <= 0:
raise ValueError("contact sheet grid must have positive cols and rows")
if not 0 <= cell_crop_ratio < 0.25:
raise ValueError("cell_crop_ratio must be in [0, 0.25)")
image = Image.open(contact_sheet).convert("RGB")
width, height = image.size
cell_w = width // cols
cell_h = height // rows
if cell_w <= 0 or cell_h <= 0:
raise ValueError(f"contact sheet is too small for {cols}x{rows}: {width}x{height}")
frames_dir.mkdir(parents=True, exist_ok=True)
views: list[Path] = []
for index in range(cols * rows):
col = index % cols
row = index // cols
left = col * cell_w
top = row * cell_h
right = width if col == cols - 1 else (col + 1) * cell_w
bottom = height if row == rows - 1 else (row + 1) * cell_h
crop_x = int((right - left) * cell_crop_ratio)
crop_y = int((bottom - top) * cell_crop_ratio)
if right - left > (2 * crop_x + 8) and bottom - top > (2 * crop_y + 8):
left += crop_x
right -= crop_x
top += crop_y
bottom -= crop_y
name = VIEW_NAMES[index] if index < len(VIEW_NAMES) else f"{index + 1:02d}_view"
path = frames_dir / f"{name}.jpg"
panel = image.crop((left, top, right, bottom))
if remove_label_bars:
panel = crop_dark_bottom_label_bar(panel)
panel.save(path, quality=95)
views.append(path)
return views
def crop_dark_bottom_label_bar(image: "Image.Image") -> "Image.Image":
import numpy as np
gray = np.asarray(image.convert("L"), dtype=np.float32) / 255.0
height, width = gray.shape
if height < 80 or width < 80:
return image
dark_fraction = np.mean(gray < 0.10, axis=1)
row_mean = np.mean(gray, axis=1)
start = height
scan_limit = int(height * 0.55)
for row in range(height - 1, scan_limit, -1):
if dark_fraction[row] > 0.55 and row_mean[row] < 0.22:
start = row
continue
if start < height:
break
if start == height:
return image
while start > scan_limit and dark_fraction[start - 1] > 0.35 and row_mean[start - 1] < 0.35:
start -= 1
bar_height = height - start
if bar_height < max(12, int(height * 0.035)):
return image
if start < int(height * 0.60):
return image
return image.crop((0, 0, width, start))
def mime_type_for_image(path: Path) -> str:
suffix = path.suffix.lower()
if suffix in {".jpg", ".jpeg"}:
return "image/jpeg"
if suffix == ".webp":
return "image/webp"
return "image/png"
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""Materialize movable extraction tasks into provisional Real2Sim object assets."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from real2sim.builtin import write_3dgs_ply, write_box_obj
from real2sim.contracts import write_json
from real2sim.image_io import read_gray, read_rgb
def main() -> int:
parser = argparse.ArgumentParser(description="Convert movable extraction task crops/masks into provisional object assets")
parser.add_argument("--package", type=Path, required=True, help="Real2Sim package root")
parser.add_argument("--tasks", type=Path, default=None, help="movable_extraction_tasks.json; defaults under package")
parser.add_argument("--output-objects-json", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--max-objects", type=int, default=3)
args = parser.parse_args()
result = materialize_movable_tasks(
package_root=args.package,
tasks_path=args.tasks or args.package / "artifacts" / "assets" / "movable_extraction_tasks.json",
output_objects_json=args.output_objects_json,
output_dir=args.output_dir,
max_objects=max(1, int(args.max_objects)),
)
print(json.dumps(result, indent=2, ensure_ascii=False))
return 0
def materialize_movable_tasks(
*,
package_root: Path,
tasks_path: Path,
output_objects_json: Path,
output_dir: Path,
max_objects: int,
) -> dict[str, Any]:
tasks_data = json.loads(tasks_path.read_text(encoding="utf-8"))
tasks = tasks_data.get("tasks", []) if isinstance(tasks_data, dict) else []
output_dir.mkdir(parents=True, exist_ok=True)
objects: list[dict[str, Any]] = []
for index, task in enumerate(tasks[:max_objects]):
if not isinstance(task, dict):
continue
asset = materialize_task_asset(package_root=package_root, output_dir=output_dir, task=task, index=index)
if asset is not None:
objects.append(asset)
payload = {
"format": "gs-playground-real2sim-provisional-objects-v1",
"objects": objects,
"quality": {
"object_count": len(objects),
"source_task_count": len(tasks),
"provisional": True,
},
"commercial_gap_notes": [
"These assets are deterministic provisional billboards/boxes from extraction crops.",
"Replace them with SAM/SAM3D object reconstructions for commercial visual fidelity.",
],
}
write_json(output_objects_json, payload)
return {"objects_json": output_objects_json.as_posix(), **payload["quality"]}
def materialize_task_asset(*, package_root: Path, output_dir: Path, task: dict[str, Any], index: int) -> dict[str, Any] | None:
crop_value = task.get("crop")
mask_value = task.get("mask")
if not crop_value or not mask_value:
return None
crop_path = package_root / str(crop_value)
mask_path = package_root / str(mask_value)
rgb = read_rgb(crop_path)
mask = read_gray(mask_path)
local_mask = local_crop_mask(mask, task.get("bbox_xyxy"), rgb.shape[:2])
if int(local_mask.sum()) < 4:
local_mask = np.ones(rgb.shape[:2], dtype=bool)
points, colors = crop_billboard_points(rgb, local_mask, max_points=6000)
if points.size == 0:
return None
label = best_prompt_label(task)
name = f"{index:02d}_{safe_token(label)}"
asset_dir = output_dir / name
gaussian = asset_dir / "gaussian.ply"
mesh = asset_dir / "mesh.obj"
extent = object_extent_from_mask(local_mask, rgb.shape[:2], label=label)
write_3dgs_ply(gaussian, points, colors, point_scale=0.01, opacity_logit=4.0)
write_box_obj(mesh, extent)
pose = pose_from_task_bbox(task.get("bbox_xyxy"))
return {
"name": name,
"label": label,
"gaussian_ply": gaussian.as_posix(),
"mesh_obj": mesh.as_posix(),
"pose": pose,
"dynamic": True,
"source_instance_id": str(task.get("id") or name),
"provisional": True,
}
def local_crop_mask(global_mask: np.ndarray, bbox: Any, crop_shape: tuple[int, int]) -> np.ndarray:
if not isinstance(bbox, list) or len(bbox) != 4:
return np.ones(crop_shape, dtype=bool)
x0, y0, x1, y1 = [int(value) for value in bbox]
cropped = global_mask[max(0, y0) : max(0, y1), max(0, x0) : max(0, x1)] > 0.5
if cropped.shape != crop_shape:
return np.ones(crop_shape, dtype=bool)
return cropped
def crop_billboard_points(rgb: np.ndarray, mask: np.ndarray, *, max_points: int) -> tuple[np.ndarray, np.ndarray]:
ys, xs = np.where(mask)
if ys.size == 0:
return np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.float32)
if ys.size > max_points:
indices = np.linspace(0, ys.size - 1, max_points, dtype=np.int64)
ys = ys[indices]
xs = xs[indices]
h, w = rgb.shape[:2]
x = ((xs.astype(np.float32) / max(w - 1, 1)) - 0.5) * 0.38
z = (0.5 - (ys.astype(np.float32) / max(h - 1, 1))) * 0.38
y = np.zeros_like(x)
points = np.stack([x, y, z], axis=1).astype(np.float32)
colors = rgb[ys, xs, :3].astype(np.float32)
return points, colors
def object_extent_from_mask(mask: np.ndarray, shape: tuple[int, int], *, label: str) -> np.ndarray:
ys, xs = np.where(mask)
if ys.size == 0:
return np.array([0.2, 0.08, 0.2], dtype=np.float32)
h, w = shape
width = max((float(xs.max() - xs.min() + 1) / max(w, 1)) * 0.38, 0.06)
height = max((float(ys.max() - ys.min() + 1) / max(h, 1)) * 0.38, 0.06)
depth = 0.08
if "bottle" in label or "cup" in label:
depth = max(width * 0.55, 0.06)
return np.array([width, depth, height], dtype=np.float32)
def pose_from_task_bbox(bbox: Any) -> dict[str, list[float]]:
if not isinstance(bbox, list) or len(bbox) != 4:
return {"position": [0.0, -0.25, 0.65], "quat_wxyz": [1.0, 0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}
x0, y0, x1, y1 = [float(value) for value in bbox]
cx = (x0 + x1) * 0.5
cy = (y0 + y1) * 0.5
position = [round((cx - 224.0) / 224.0 * 0.9, 6), -0.35, round((448.0 - cy) / 448.0 * 0.65 + 0.25, 6)]
return {"position": position, "quat_wxyz": [1.0, 0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}
def best_prompt_label(task: dict[str, Any]) -> str:
prompts = task.get("prompt_candidates")
if isinstance(prompts, list) and prompts:
return str(prompts[0])
return "movable_object"
def safe_token(value: str) -> str:
return "".join(ch if ch.isalnum() else "_" for ch in value.lower()).strip("_") or "object"
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,382 @@
#!/usr/bin/env python3
"""Filter OBJ mesh fragments before Real2Sim texture baking."""
from __future__ import annotations
import argparse
import json
from collections import defaultdict, deque
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import numpy as np
@dataclass(frozen=True)
class ObjFace:
vertices: tuple[int, int, int]
texcoords: tuple[int, int, int]
@dataclass(frozen=True)
class ObjMesh:
vertices: np.ndarray
colors: np.ndarray
texcoords: np.ndarray
faces: tuple[ObjFace, ...]
def main() -> int:
parser = argparse.ArgumentParser(description="Filter small disconnected OBJ mesh fragments")
parser.add_argument("--input-obj", type=Path, required=True)
parser.add_argument("--output-obj", type=Path, required=True)
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--keep-largest", type=int, default=1, help="Keep this many largest connected components")
parser.add_argument("--min-faces", type=int, default=0, help="Also keep components with at least this many faces")
parser.add_argument("--min-area-ratio", type=float, default=0.0, help="Also keep components with area >= ratio * total area")
parser.add_argument("--max-face-area-percentile", type=float, default=100.0, help="Drop triangles above this area percentile")
parser.add_argument("--max-edge-length-percentile", type=float, default=100.0, help="Drop triangles whose longest edge is above this percentile")
parser.add_argument("--min-centroid-x-percentile", type=float, default=0.0, help="Drop triangles whose centroid x is below this percentile")
parser.add_argument("--max-centroid-x-percentile", type=float, default=100.0, help="Drop triangles whose centroid x is above this percentile")
parser.add_argument("--min-centroid-y-percentile", type=float, default=0.0, help="Drop triangles whose centroid y is below this percentile")
parser.add_argument("--max-centroid-y-percentile", type=float, default=100.0, help="Drop triangles whose centroid y is above this percentile")
parser.add_argument("--min-centroid-z-percentile", type=float, default=0.0, help="Drop triangles whose centroid z is below this percentile")
parser.add_argument("--max-centroid-z-percentile", type=float, default=100.0, help="Drop triangles whose centroid z is above this percentile")
args = parser.parse_args()
report = cleanup_obj_mesh(
input_obj=args.input_obj,
output_obj=args.output_obj,
keep_largest=max(int(args.keep_largest), 0),
min_faces=max(int(args.min_faces), 0),
min_area_ratio=max(float(args.min_area_ratio), 0.0),
max_face_area_percentile=float(args.max_face_area_percentile),
max_edge_length_percentile=float(args.max_edge_length_percentile),
centroid_percentile_bounds={
"x": (float(args.min_centroid_x_percentile), float(args.max_centroid_x_percentile)),
"y": (float(args.min_centroid_y_percentile), float(args.max_centroid_y_percentile)),
"z": (float(args.min_centroid_z_percentile), float(args.max_centroid_z_percentile)),
},
)
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(json.dumps(report, indent=2, ensure_ascii=False))
return 0
def cleanup_obj_mesh(
*,
input_obj: Path,
output_obj: Path,
keep_largest: int = 1,
min_faces: int = 0,
min_area_ratio: float = 0.0,
max_face_area_percentile: float = 100.0,
max_edge_length_percentile: float = 100.0,
centroid_percentile_bounds: dict[str, tuple[float, float]] | None = None,
) -> dict[str, Any]:
mesh = read_obj_mesh(input_obj)
components = connected_face_components(mesh)
face_areas = triangle_areas(mesh)
face_edges = triangle_max_edges(mesh)
centroids = triangle_centroids(mesh)
total_area = float(np.sum(face_areas))
area_threshold = percentile_threshold(face_areas, max_face_area_percentile)
edge_threshold = percentile_threshold(face_edges, max_edge_length_percentile)
centroid_filters = centroid_axis_filters(centroids, centroid_percentile_bounds)
ranked = sorted(
(
{
"component": idx,
"face_count": len(face_indices),
"area": float(np.sum(face_areas[np.asarray(face_indices, dtype=np.int64)])),
"faces": face_indices,
}
for idx, face_indices in enumerate(components)
),
key=lambda item: (item["face_count"], item["area"]),
reverse=True,
)
keep_component_ids: set[int] = set()
for item in ranked[:keep_largest]:
keep_component_ids.add(int(item["component"]))
for item in ranked:
area_ratio = (float(item["area"]) / total_area) if total_area > 0 else 0.0
if (min_faces > 0 and int(item["face_count"]) >= min_faces) or (
min_area_ratio > 0.0 and area_ratio >= min_area_ratio
):
keep_component_ids.add(int(item["component"]))
kept_faces_before_quality = [
face_index
for component_id, face_indices in enumerate(components)
if component_id in keep_component_ids
for face_index in face_indices
]
kept_faces = [
face_index
for face_index in kept_faces_before_quality
if (area_threshold is None or face_areas[int(face_index)] <= area_threshold)
and (edge_threshold is None or face_edges[int(face_index)] <= edge_threshold)
and centroid_face_allowed(centroids[int(face_index)], centroid_filters)
]
cleaned = remap_mesh(mesh, sorted(kept_faces))
output_obj.parent.mkdir(parents=True, exist_ok=True)
write_obj_mesh(cleaned, output_obj)
kept_area = float(np.sum(face_areas[np.asarray(kept_faces, dtype=np.int64)])) if kept_faces else 0.0
report = {
"format": "gs-playground-real2sim-obj-mesh-cleanup-report-v1",
"input_obj": input_obj.resolve().as_posix(),
"output_obj": output_obj.resolve().as_posix(),
"input_vertex_count": int(mesh.vertices.shape[0]),
"input_texcoord_count": int(mesh.texcoords.shape[0]),
"input_face_count": int(len(mesh.faces)),
"component_count": int(len(components)),
"kept_component_count": int(len(keep_component_ids)),
"component_kept_face_count": int(len(kept_faces_before_quality)),
"kept_face_count": int(len(kept_faces)),
"removed_face_count": int(len(mesh.faces) - len(kept_faces)),
"quality_removed_face_count": int(len(kept_faces_before_quality) - len(kept_faces)),
"centroid_filter_removed_face_count": int(
sum(
1
for face_index in kept_faces_before_quality
if not centroid_face_allowed(centroids[int(face_index)], centroid_filters)
)
),
"kept_area_ratio": round((kept_area / total_area) if total_area > 0 else 0.0, 6),
"keep_largest": int(keep_largest),
"min_faces": int(min_faces),
"min_area_ratio": float(min_area_ratio),
"max_face_area_percentile": float(max_face_area_percentile),
"max_edge_length_percentile": float(max_edge_length_percentile),
"face_area_threshold": float(area_threshold) if area_threshold is not None else None,
"edge_length_threshold": float(edge_threshold) if edge_threshold is not None else None,
"centroid_filters": centroid_filters,
"largest_components": [
{
"component": int(item["component"]),
"face_count": int(item["face_count"]),
"area_ratio": round((float(item["area"]) / total_area) if total_area > 0 else 0.0, 6),
}
for item in ranked[:10]
],
}
return report
def connected_face_components(mesh: ObjMesh) -> list[list[int]]:
vertex_to_faces: dict[int, list[int]] = defaultdict(list)
for face_index, face in enumerate(mesh.faces):
for vertex in face.vertices:
vertex_to_faces[int(vertex)].append(face_index)
visited = np.zeros(len(mesh.faces), dtype=bool)
components: list[list[int]] = []
for start in range(len(mesh.faces)):
if visited[start]:
continue
queue: deque[int] = deque([start])
visited[start] = True
component: list[int] = []
while queue:
face_index = queue.popleft()
component.append(face_index)
for vertex in mesh.faces[face_index].vertices:
for neighbor in vertex_to_faces[int(vertex)]:
if not visited[neighbor]:
visited[neighbor] = True
queue.append(neighbor)
components.append(component)
return components
def triangle_areas(mesh: ObjMesh) -> np.ndarray:
if not mesh.faces:
return np.zeros((0,), dtype=np.float64)
indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64)
points = mesh.vertices[indices].astype(np.float64)
areas = np.linalg.norm(np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]), axis=1) * 0.5
return areas
def triangle_max_edges(mesh: ObjMesh) -> np.ndarray:
if not mesh.faces:
return np.zeros((0,), dtype=np.float64)
indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64)
points = mesh.vertices[indices].astype(np.float64)
e01 = np.linalg.norm(points[:, 0] - points[:, 1], axis=1)
e12 = np.linalg.norm(points[:, 1] - points[:, 2], axis=1)
e20 = np.linalg.norm(points[:, 2] - points[:, 0], axis=1)
return np.maximum(np.maximum(e01, e12), e20)
def triangle_centroids(mesh: ObjMesh) -> np.ndarray:
if not mesh.faces:
return np.zeros((0, 3), dtype=np.float64)
indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64)
points = mesh.vertices[indices].astype(np.float64)
return points.mean(axis=1)
def percentile_threshold(values: np.ndarray, percentile: float) -> float | None:
if values.size == 0 or percentile >= 100.0:
return None
clamped = min(max(float(percentile), 0.0), 100.0)
return float(np.percentile(values, clamped))
def centroid_axis_filters(
centroids: np.ndarray,
percentile_bounds: dict[str, tuple[float, float]] | None,
) -> dict[str, dict[str, float | None]]:
axes = {"x": 0, "y": 1, "z": 2}
filters: dict[str, dict[str, float | None]] = {}
if centroids.size == 0:
return {
axis: {"min_percentile": None, "max_percentile": None, "min_value": None, "max_value": None}
for axis in axes
}
percentile_bounds = percentile_bounds or {}
for axis, index in axes.items():
lower, upper = percentile_bounds.get(axis, (0.0, 100.0))
lower = min(max(float(lower), 0.0), 100.0)
upper = min(max(float(upper), 0.0), 100.0)
if lower > upper:
lower, upper = upper, lower
values = centroids[:, index]
min_value = float(np.percentile(values, lower)) if lower > 0.0 else None
max_value = float(np.percentile(values, upper)) if upper < 100.0 else None
filters[axis] = {
"min_percentile": lower if min_value is not None else None,
"max_percentile": upper if max_value is not None else None,
"min_value": min_value,
"max_value": max_value,
}
return filters
def centroid_face_allowed(centroid: np.ndarray, filters: dict[str, dict[str, float | None]]) -> bool:
for axis, index in {"x": 0, "y": 1, "z": 2}.items():
bounds = filters.get(axis, {})
min_value = bounds.get("min_value")
max_value = bounds.get("max_value")
value = float(centroid[index])
if min_value is not None and value < float(min_value):
return False
if max_value is not None and value > float(max_value):
return False
return True
def remap_mesh(mesh: ObjMesh, face_indices: list[int]) -> ObjMesh:
vertex_map: dict[int, int] = {}
texcoord_map: dict[int, int] = {}
vertices: list[np.ndarray] = []
colors: list[np.ndarray] = []
texcoords: list[np.ndarray] = []
faces: list[ObjFace] = []
for face_index in face_indices:
face = mesh.faces[int(face_index)]
new_vertices: list[int] = []
new_texcoords: list[int] = []
for vertex in face.vertices:
if vertex not in vertex_map:
vertex_map[vertex] = len(vertices)
vertices.append(mesh.vertices[vertex])
colors.append(mesh.colors[vertex])
new_vertices.append(vertex_map[vertex])
for texcoord in face.texcoords:
if texcoord not in texcoord_map:
texcoord_map[texcoord] = len(texcoords)
texcoords.append(mesh.texcoords[texcoord])
new_texcoords.append(texcoord_map[texcoord])
faces.append(ObjFace(vertices=tuple(new_vertices), texcoords=tuple(new_texcoords))) # type: ignore[arg-type]
return ObjMesh(
vertices=np.asarray(vertices, dtype=np.float32),
colors=np.asarray(colors, dtype=np.float32),
texcoords=np.asarray(texcoords, dtype=np.float32),
faces=tuple(faces),
)
def read_obj_mesh(path: Path) -> ObjMesh:
vertices: list[tuple[float, float, float]] = []
colors: list[tuple[float, float, float]] = []
texcoords: list[tuple[float, float]] = []
faces: list[ObjFace] = []
for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines():
if raw.startswith("v "):
parts = raw.split()
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
if len(parts) >= 7:
colors.append((float(parts[4]), float(parts[5]), float(parts[6])))
else:
colors.append((0.5, 0.5, 0.5))
elif raw.startswith("vt "):
parts = raw.split()
if len(parts) >= 3:
texcoords.append((float(parts[1]), float(parts[2])))
elif raw.startswith("f "):
refs = [parse_face_ref(token) for token in raw.split()[1:]]
if len(refs) < 3:
continue
for idx in range(1, len(refs) - 1):
tri = (refs[0], refs[idx], refs[idx + 1])
faces.append(
ObjFace(
vertices=tuple(item[0] for item in tri),
texcoords=tuple(item[1] if item[1] is not None else item[0] for item in tri),
)
)
if not texcoords:
texcoords = [(0.0, 0.0) for _ in vertices]
return ObjMesh(
vertices=np.asarray(vertices, dtype=np.float32),
colors=np.clip(np.asarray(colors, dtype=np.float32), 0.0, 1.0),
texcoords=np.asarray(texcoords, dtype=np.float32),
faces=tuple(faces),
)
def parse_face_ref(token: str) -> tuple[int, int | None]:
parts = token.split("/")
vertex = int(parts[0]) - 1
texcoord = int(parts[1]) - 1 if len(parts) >= 2 and parts[1] else None
return vertex, texcoord
def write_obj_mesh(mesh: ObjMesh, output_obj: Path) -> None:
material_name = output_obj.stem + "_mat"
material_file = output_obj.with_suffix(".mtl")
with material_file.open("w", encoding="utf-8") as dst:
dst.write("# Generated Real2Sim cleaned OBJ material\n")
dst.write(f"newmtl {material_name}\n")
dst.write("Ka 1.000000 1.000000 1.000000\n")
dst.write("Kd 1.000000 1.000000 1.000000\n")
dst.write("Ks 0.030000 0.030000 0.030000\n")
dst.write("Ns 8.000000\n")
dst.write("d 1.000000\n")
dst.write("illum 2\n")
with output_obj.open("w", encoding="utf-8") as dst:
dst.write("# Real2Sim cleaned scene mesh\n")
dst.write(f"mtllib {material_file.name}\n")
dst.write("o real2sim_cleaned_scene\n")
for vertex, color in zip(mesh.vertices, mesh.colors):
dst.write(
f"v {vertex[0]:.9g} {vertex[1]:.9g} {vertex[2]:.9g} "
f"{color[0]:.6f} {color[1]:.6f} {color[2]:.6f}\n"
)
for texcoord in mesh.texcoords:
dst.write(f"vt {texcoord[0]:.7g} {texcoord[1]:.7g}\n")
dst.write(f"usemtl {material_name}\n")
for face in mesh.faces:
dst.write(
"f "
f"{face.vertices[0] + 1}/{face.texcoords[0] + 1} "
f"{face.vertices[1] + 1}/{face.texcoords[1] + 1} "
f"{face.vertices[2] + 1}/{face.texcoords[2] + 1}\n"
)
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,265 @@
#!/usr/bin/env python3
"""Add coarse room/tabletop completion planes to a reconstructed OBJ scene.
The COLMAP/Delaunay mesh path often produces visually useful fragments but
leaves large holes in walls, tabletops, or floors. This utility keeps the
original OBJ intact and appends a conservative, quantile-bounded planar shell
so MotrixSim has a continuous scene scaffold for inspection and collision.
"""
from __future__ import annotations
import argparse
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable
@dataclass(frozen=True)
class Bounds:
low: tuple[float, float, float]
high: tuple[float, float, float]
@dataclass(frozen=True)
class Plane:
name: str
axis: int
value: float
u_axis: int
v_axis: int
u_min: float
u_max: float
v_min: float
v_max: float
material: str
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Append quantile-bounded scene completion planes to an OBJ")
parser.add_argument("--input-obj", type=Path, required=True)
parser.add_argument("--output-obj", type=Path, required=True)
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--low-quantile", type=float, default=0.05)
parser.add_argument("--high-quantile", type=float, default=0.95)
parser.add_argument("--margin-ratio", type=float, default=0.03)
parser.add_argument("--up-axis", choices=("x", "y", "z"), default="z")
parser.add_argument("--grid", type=int, default=24, help="Subdivisions per plane axis")
parser.add_argument(
"--planes",
default="floor,ceiling,left,right,back,front",
help="Comma-separated planes to append: floor,ceiling,left,right,back,front",
)
args = parser.parse_args(argv)
vertices, texcoord_count = read_obj_counts(args.input_obj)
if len(vertices) < 3:
raise SystemExit(f"OBJ has too few vertices for scene completion: {args.input_obj}")
low_q = clamp(args.low_quantile, 0.0, 0.49)
high_q = clamp(args.high_quantile, 0.51, 1.0)
if low_q >= high_q:
raise SystemExit("--low-quantile must be below --high-quantile")
bounds = quantile_bounds(vertices, low_q, high_q, margin_ratio=max(args.margin_ratio, 0.0))
up_axis = "xyz".index(args.up_axis)
plane_names = tuple(name.strip().lower() for name in args.planes.split(",") if name.strip())
planes = build_planes(bounds, up_axis=up_axis, names=plane_names)
if not planes:
raise SystemExit("No completion planes selected")
args.output_obj.parent.mkdir(parents=True, exist_ok=True)
mtl_path = args.output_obj.with_name(f"{args.output_obj.stem}_completion.mtl")
append_completion(
args.input_obj,
args.output_obj,
mtl_path,
vertices_start=len(vertices),
texcoords_start=texcoord_count,
planes=planes,
grid=max(args.grid, 1),
)
report = {
"format": "gs-playground-real2sim-obj-scene-completion-report-v1",
"input_obj": args.input_obj.as_posix(),
"output_obj": args.output_obj.as_posix(),
"material_library": mtl_path.name,
"input_vertex_count": len(vertices),
"input_texcoord_count": texcoord_count,
"bounds_low": list(bounds.low),
"bounds_high": list(bounds.high),
"low_quantile": low_q,
"high_quantile": high_q,
"margin_ratio": max(args.margin_ratio, 0.0),
"up_axis": args.up_axis,
"grid": max(args.grid, 1),
"plane_count": len(planes),
"planes": [plane.name for plane in planes],
"added_vertex_count": len(planes) * (max(args.grid, 1) + 1) ** 2,
"added_texcoord_count": len(planes) * (max(args.grid, 1) + 1) ** 2,
"added_face_count": len(planes) * max(args.grid, 1) * max(args.grid, 1) * 2,
}
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(json.dumps(report, indent=2, ensure_ascii=False))
return 0
def read_obj_counts(path: Path) -> tuple[list[tuple[float, float, float]], int]:
vertices: list[tuple[float, float, float]] = []
texcoord_count = 0
with path.open("r", encoding="utf-8", errors="ignore") as handle:
for line in handle:
if line.startswith("vt "):
texcoord_count += 1
continue
if not line.startswith("v "):
continue
parts = line.split()
if len(parts) < 4:
continue
try:
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
except ValueError:
continue
return vertices, texcoord_count
def quantile_bounds(
vertices: Iterable[tuple[float, float, float]],
low_quantile: float,
high_quantile: float,
*,
margin_ratio: float,
) -> Bounds:
columns = list(zip(*vertices))
lows: list[float] = []
highs: list[float] = []
for values in columns:
ordered = sorted(values)
low = percentile(ordered, low_quantile)
high = percentile(ordered, high_quantile)
span = max(high - low, 1e-6)
lows.append(low - span * margin_ratio)
highs.append(high + span * margin_ratio)
return Bounds(tuple(lows), tuple(highs))
def percentile(values: list[float], q: float) -> float:
if not values:
raise ValueError("percentile requires values")
if len(values) == 1:
return values[0]
pos = q * (len(values) - 1)
lo = int(pos)
hi = min(lo + 1, len(values) - 1)
frac = pos - lo
return values[lo] * (1.0 - frac) + values[hi] * frac
def build_planes(bounds: Bounds, *, up_axis: int, names: tuple[str, ...]) -> list[Plane]:
axes = {0, 1, 2}
horizontal = [axis for axis in axes if axis != up_axis]
a0, a1 = horizontal
low, high = bounds.low, bounds.high
specs = {
"floor": Plane("floor", up_axis, low[up_axis], a0, a1, low[a0], high[a0], low[a1], high[a1], "real2sim_completion_floor"),
"ceiling": Plane("ceiling", up_axis, high[up_axis], a0, a1, low[a0], high[a0], low[a1], high[a1], "real2sim_completion_ceiling"),
"left": Plane("left", a0, low[a0], a1, up_axis, low[a1], high[a1], low[up_axis], high[up_axis], "real2sim_completion_wall"),
"right": Plane("right", a0, high[a0], a1, up_axis, low[a1], high[a1], low[up_axis], high[up_axis], "real2sim_completion_wall"),
"back": Plane("back", a1, low[a1], a0, up_axis, low[a0], high[a0], low[up_axis], high[up_axis], "real2sim_completion_wall"),
"front": Plane("front", a1, high[a1], a0, up_axis, low[a0], high[a0], low[up_axis], high[up_axis], "real2sim_completion_wall"),
}
return [specs[name] for name in names if name in specs]
def append_completion(
input_obj: Path,
output_obj: Path,
mtl_path: Path,
*,
vertices_start: int,
texcoords_start: int,
planes: list[Plane],
grid: int,
) -> None:
source = input_obj.read_text(encoding="utf-8", errors="ignore")
output = [source]
if not source.endswith("\n"):
output.append("\n")
output.append(f"mtllib {mtl_path.name}\n")
output.append("o real2sim_scene_completion\n")
vertex_index = vertices_start
for plane in planes:
output.append(f"g real2sim_completion_{plane.name}\n")
output.append(f"usemtl {plane.material}\n")
vertex_start = vertex_index + 1
texcoord_start = texcoords_start + 1
for iv in range(grid + 1):
v_fraction = iv / grid
v = lerp(plane.v_min, plane.v_max, v_fraction)
for iu in range(grid + 1):
u_fraction = iu / grid
u = lerp(plane.u_min, plane.u_max, u_fraction)
coords = [0.0, 0.0, 0.0]
coords[plane.axis] = plane.value
coords[plane.u_axis] = u
coords[plane.v_axis] = v
output.append(f"v {coords[0]:.8f} {coords[1]:.8f} {coords[2]:.8f}\n")
output.append(f"vt {u_fraction:.8f} {v_fraction:.8f}\n")
for iv in range(grid):
for iu in range(grid):
a = vertex_start + iv * (grid + 1) + iu
b = a + 1
c = a + (grid + 1)
d = c + 1
ta = texcoord_start + iv * (grid + 1) + iu
tb = ta + 1
tc = ta + (grid + 1)
td = tc + 1
output.append(f"f {a}/{ta} {b}/{tb} {d}/{td}\n")
output.append(f"f {a}/{ta} {d}/{td} {c}/{tc}\n")
vertex_index += (grid + 1) * (grid + 1)
texcoords_start += (grid + 1) * (grid + 1)
output_obj.write_text("".join(output), encoding="utf-8")
write_completion_mtl(mtl_path)
def write_completion_mtl(path: Path) -> None:
path.write_text(
"\n".join(
[
"# Generated Real2Sim scene completion materials",
"newmtl real2sim_completion_floor",
"Kd 0.420000 0.410000 0.390000",
"Ka 0.120000 0.120000 0.120000",
"Ks 0.020000 0.020000 0.020000",
"Ns 8.000000",
"",
"newmtl real2sim_completion_wall",
"Kd 0.680000 0.650000 0.590000",
"Ka 0.150000 0.145000 0.135000",
"Ks 0.015000 0.015000 0.015000",
"Ns 6.000000",
"",
"newmtl real2sim_completion_ceiling",
"Kd 0.720000 0.700000 0.660000",
"Ka 0.160000 0.155000 0.145000",
"Ks 0.010000 0.010000 0.010000",
"Ns 4.000000",
"",
]
),
encoding="utf-8",
)
def lerp(a: float, b: float, t: float) -> float:
return a * (1.0 - t) + b * t
def clamp(value: float, low: float, high: float) -> float:
return min(max(value, low), high)
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,461 @@
#!/usr/bin/env python3
"""Extract coarse room/scene structure cues from an OBJ mesh."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
import numpy as np
def main() -> int:
parser = argparse.ArgumentParser(description="Analyze axis-aligned scene structure candidates in an OBJ mesh")
parser.add_argument("--input-obj", type=Path, required=True)
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--up-axis", choices=("x", "y", "z"), default="z")
parser.add_argument("--normal-threshold", type=float, default=0.78)
parser.add_argument("--support-normal-threshold", type=float, default=0.82)
args = parser.parse_args()
report = analyze_obj_structure(
input_obj=args.input_obj,
up_axis=args.up_axis,
normal_threshold=max(min(float(args.normal_threshold), 1.0), 0.0),
support_normal_threshold=max(min(float(args.support_normal_threshold), 1.0), 0.0),
)
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(json.dumps(report, indent=2, ensure_ascii=False))
return 0
def analyze_obj_structure(
*,
input_obj: Path,
up_axis: str = "z",
normal_threshold: float = 0.78,
support_normal_threshold: float = 0.82,
) -> dict[str, Any]:
vertices, faces = read_obj_triangles(input_obj)
if len(vertices) < 3 or len(faces) < 1:
raise SystemExit(f"OBJ has no usable triangle mesh: {input_obj}")
points = vertices[faces]
normals = triangle_normals(points)
areas = triangle_areas(points)
centroids = points.mean(axis=1)
total_area = float(np.sum(areas))
up_index = "xyz".index(up_axis)
axis_reports = [
axis_structure_report(
axis=index,
name=name,
normals=normals,
centroids=centroids,
areas=areas,
total_area=total_area,
threshold=normal_threshold,
)
for index, name in enumerate("xyz")
]
support = support_plane_candidates(
up_axis=up_index,
normals=normals,
centroids=centroids,
areas=areas,
total_area=total_area,
threshold=support_normal_threshold,
)
bounds = {
"min": [float(value) for value in vertices.min(axis=0)],
"max": [float(value) for value in vertices.max(axis=0)],
"p05": [float(np.percentile(vertices[:, idx], 5)) for idx in range(3)],
"p95": [float(np.percentile(vertices[:, idx], 95)) for idx in range(3)],
}
semantic_planes = semantic_plane_candidates(
up_axis=up_index,
normals=normals,
centroids=centroids,
areas=areas,
bounds=bounds,
total_area=total_area,
threshold=normal_threshold,
)
semantic_planes = add_recovered_floor_candidate(
semantic_planes,
up_axis=up_index,
normals=normals,
centroids=centroids,
areas=areas,
bounds=bounds,
total_area=total_area,
threshold=support_normal_threshold,
)
structure_score = min(
1.0,
sum(float(axis["area_ratio"]) for axis in axis_reports) / 3.0
+ min(float(support["support_area_ratio"]) * 0.5, 0.25),
)
semantic_score = semantic_structure_score(semantic_planes)
return {
"format": "gs-playground-real2sim-obj-structure-report-v1",
"input_obj": input_obj.resolve().as_posix(),
"vertex_count": int(vertices.shape[0]),
"face_count": int(faces.shape[0]),
"total_area": total_area,
"up_axis": up_axis,
"bounds": bounds,
"axis_aligned_planes": axis_reports,
"support_planes": support,
"semantic_planes": semantic_planes,
"structure_score": round(float(structure_score), 6),
"commercial_readiness": {
"structure_score": round(float(structure_score), 6),
"semantic_structure_score": round(float(semantic_score), 6),
"has_floor": any(plane["semantic_label"] == "floor" for plane in semantic_planes),
"has_wall": any(plane["semantic_label"].startswith("wall_") for plane in semantic_planes),
"has_support_surface": any(plane["semantic_label"] == "support_surface" for plane in semantic_planes),
"remaining_blockers": [
"semantic instance segmentation for movable assets",
"watertight hole filling and room-boundary regularization",
"collision/nav mesh generation from semantic planes",
"physically meaningful material parameters",
],
},
"commercial_gap_notes": [
"This report now adds coarse semantic structure candidates, but it is not full room understanding.",
"Commercial-grade Real2Sim still needs instance semantics, watertight reconstruction, collision/nav mesh derivation, and physical material inference.",
],
}
def axis_structure_report(
*,
axis: int,
name: str,
normals: np.ndarray,
centroids: np.ndarray,
areas: np.ndarray,
total_area: float,
threshold: float,
) -> dict[str, Any]:
aligned = np.abs(normals[:, axis]) >= threshold
area = float(np.sum(areas[aligned])) if np.any(aligned) else 0.0
values = centroids[aligned, axis] if np.any(aligned) else np.asarray([], dtype=np.float64)
return {
"axis": name,
"aligned_face_count": int(np.count_nonzero(aligned)),
"area": area,
"area_ratio": round((area / total_area) if total_area > 0 else 0.0, 6),
"low_plane": plane_summary(values, areas[aligned], quantile=5),
"high_plane": plane_summary(values, areas[aligned], quantile=95),
}
def support_plane_candidates(
*,
up_axis: int,
normals: np.ndarray,
centroids: np.ndarray,
areas: np.ndarray,
total_area: float,
threshold: float,
) -> dict[str, Any]:
upward = np.abs(normals[:, up_axis]) >= threshold
values = centroids[upward, up_axis] if np.any(upward) else np.asarray([], dtype=np.float64)
support_areas = areas[upward] if np.any(upward) else np.asarray([], dtype=np.float64)
low = plane_summary(values, support_areas, quantile=10)
high = plane_summary(values, support_areas, quantile=75)
support_area = float(np.sum(support_areas))
return {
"up_axis": "xyz"[up_axis],
"support_face_count": int(np.count_nonzero(upward)),
"support_area": support_area,
"support_area_ratio": round((support_area / total_area) if total_area > 0 else 0.0, 6),
"floor_like": low,
"tabletop_or_shelf_like": high,
}
def semantic_plane_candidates(
*,
up_axis: int,
normals: np.ndarray,
centroids: np.ndarray,
areas: np.ndarray,
bounds: dict[str, list[float]],
total_area: float,
threshold: float,
) -> list[dict[str, Any]]:
planes: list[dict[str, Any]] = []
for axis, name in enumerate("xyz"):
axis_span = max(float(bounds["p95"][axis] - bounds["p05"][axis]), 1e-6)
clusters = clustered_axis_planes(
axis=axis,
normals=normals,
centroids=centroids,
areas=areas,
threshold=threshold,
cluster_tolerance=max(axis_span * 0.035, 0.04),
)
for cluster in clusters:
label = semantic_label_for_plane(axis=axis, up_axis=up_axis, coordinate=cluster["coordinate"], bounds=bounds)
if label == "vertical_partition" and cluster["area_ratio"] >= 0.05:
label = "wall_visible"
if label == "support_surface" and cluster["area_ratio"] < 0.01:
continue
if label.startswith("wall_") and cluster["area_ratio"] < 0.015:
continue
planes.append(
{
"semantic_label": label,
"axis": name,
"coordinate": round(float(cluster["coordinate"]), 6),
"area": cluster["area"],
"area_ratio": round(float(cluster["area_ratio"]), 6),
"face_count": cluster["face_count"],
"extent": cluster["extent"],
"collision_proxy_hint": collision_proxy_hint(
label=label,
axis=axis,
coordinate=float(cluster["coordinate"]),
extent=cluster["extent"],
bounds=bounds,
),
}
)
planes.sort(key=lambda item: (semantic_priority(item["semantic_label"]), -float(item["area_ratio"])))
return planes[:16]
def add_recovered_floor_candidate(
planes: list[dict[str, Any]],
*,
up_axis: int,
normals: np.ndarray,
centroids: np.ndarray,
areas: np.ndarray,
bounds: dict[str, list[float]],
total_area: float,
threshold: float,
) -> list[dict[str, Any]]:
if any(plane["semantic_label"] == "floor" for plane in planes):
return planes
candidate = recovered_floor_candidate(
up_axis=up_axis,
normals=normals,
centroids=centroids,
areas=areas,
bounds=bounds,
total_area=total_area,
threshold=threshold,
)
if candidate is None:
return planes
updated = [candidate, *planes]
updated.sort(key=lambda item: (semantic_priority(item["semantic_label"]), -float(item["area_ratio"])))
return updated[:16]
def recovered_floor_candidate(
*,
up_axis: int,
normals: np.ndarray,
centroids: np.ndarray,
areas: np.ndarray,
bounds: dict[str, list[float]],
total_area: float,
threshold: float,
) -> dict[str, Any] | None:
upward = np.abs(normals[:, up_axis]) >= threshold
if not np.any(upward):
return None
up_values = centroids[upward, up_axis]
up_areas = areas[upward]
low = float(bounds["p05"][up_axis])
high = float(bounds["p95"][up_axis])
span = max(high - low, 1e-6)
coordinate = float(np.percentile(up_values, 10))
if coordinate > low + span * 0.18:
return None
near = np.abs(up_values - coordinate) <= max(span * 0.025, 0.035)
if not np.any(near):
return None
near_points = centroids[upward][near]
near_area = float(np.sum(up_areas[near]))
area_ratio = near_area / total_area if total_area > 0 else 0.0
if area_ratio < 0.012 or near_points.shape[0] < 80:
return None
extent = {
name: [float(np.percentile(near_points[:, idx], 5)), float(np.percentile(near_points[:, idx], 95))]
for idx, name in enumerate("xyz")
}
return {
"semantic_label": "floor",
"axis": "xyz"[up_axis],
"coordinate": round(coordinate, 6),
"area": near_area,
"area_ratio": round(float(area_ratio), 6),
"face_count": int(near_points.shape[0]),
"extent": extent,
"recovery_method": "low_horizontal_support_quantile",
"recovery_confidence": round(min(1.0, area_ratio / 0.05), 6),
"collision_proxy_hint": collision_proxy_hint(
label="floor",
axis=up_axis,
coordinate=coordinate,
extent=extent,
bounds=bounds,
),
}
def clustered_axis_planes(
*,
axis: int,
normals: np.ndarray,
centroids: np.ndarray,
areas: np.ndarray,
threshold: float,
cluster_tolerance: float,
) -> list[dict[str, Any]]:
aligned = np.abs(normals[:, axis]) >= threshold
if not np.any(aligned):
return []
aligned_centroids = centroids[aligned]
aligned_areas = areas[aligned]
order = np.argsort(aligned_centroids[:, axis])
sorted_centroids = aligned_centroids[order]
sorted_areas = aligned_areas[order]
clusters: list[dict[str, Any]] = []
start = 0
for index in range(1, sorted_centroids.shape[0] + 1):
boundary = index == sorted_centroids.shape[0]
if not boundary:
boundary = float(sorted_centroids[index, axis] - sorted_centroids[index - 1, axis]) > cluster_tolerance
if boundary:
points = sorted_centroids[start:index]
weights = sorted_areas[start:index]
area = float(np.sum(weights))
if points.size > 0 and area > 0:
clusters.append(
{
"coordinate": float(np.average(points[:, axis], weights=weights)),
"area": area,
"area_ratio": area / float(np.sum(areas)) if np.sum(areas) > 0 else 0.0,
"face_count": int(points.shape[0]),
"extent": {
name: [float(np.percentile(points[:, idx], 5)), float(np.percentile(points[:, idx], 95))]
for idx, name in enumerate("xyz")
},
}
)
start = index
clusters.sort(key=lambda item: item["area"], reverse=True)
return clusters[:8]
def semantic_label_for_plane(*, axis: int, up_axis: int, coordinate: float, bounds: dict[str, list[float]]) -> str:
low = float(bounds["p05"][axis])
high = float(bounds["p95"][axis])
span = max(high - low, 1e-6)
if axis == up_axis:
if coordinate <= low + span * 0.12:
return "floor"
if coordinate >= high - span * 0.12:
return "ceiling"
return "support_surface"
if coordinate <= low + span * 0.18:
return f"wall_{'xyz'[axis]}_low"
if coordinate >= high - span * 0.18:
return f"wall_{'xyz'[axis]}_high"
return "vertical_partition"
def collision_proxy_hint(
*,
label: str,
axis: int,
coordinate: float,
extent: dict[str, list[float]],
bounds: dict[str, list[float]],
) -> dict[str, Any]:
center = [(float(extent[name][0]) + float(extent[name][1])) * 0.5 for name in "xyz"]
size = [max(float(extent[name][1]) - float(extent[name][0]), 0.02) for name in "xyz"]
center[axis] = coordinate
size[axis] = max(float(bounds["p95"][axis] - bounds["p05"][axis]) * 0.01, 0.02)
return {
"type": "box",
"label": label,
"center": [round(value, 6) for value in center],
"size": [round(value, 6) for value in size],
}
def semantic_priority(label: str) -> int:
if label == "floor":
return 0
if label.startswith("wall_"):
return 1
if label == "support_surface":
return 2
if label == "ceiling":
return 3
return 4
def semantic_structure_score(planes: list[dict[str, Any]]) -> float:
has_floor = any(plane["semantic_label"] == "floor" for plane in planes)
wall_count = sum(1 for plane in planes if plane["semantic_label"].startswith("wall_"))
support_count = sum(1 for plane in planes if plane["semantic_label"] == "support_surface")
return min(1.0, (0.35 if has_floor else 0.0) + min(wall_count, 4) * 0.12 + min(support_count, 3) * 0.08)
def plane_summary(values: np.ndarray, areas: np.ndarray, *, quantile: float) -> dict[str, Any] | None:
if values.size == 0:
return None
value = float(np.percentile(values, quantile))
span = max(float(np.percentile(values, 95) - np.percentile(values, 5)), 1e-6)
near = np.abs(values - value) <= max(span * 0.04, 0.02)
near_area = float(np.sum(areas[near])) if np.any(near) else 0.0
return {
"coordinate": value,
"quantile": float(quantile),
"near_face_count": int(np.count_nonzero(near)),
"near_area": near_area,
}
def read_obj_triangles(path: Path) -> tuple[np.ndarray, np.ndarray]:
vertices: list[tuple[float, float, float]] = []
faces: list[tuple[int, int, int]] = []
for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines():
if raw.startswith("v "):
parts = raw.split()
if len(parts) >= 4:
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
elif raw.startswith("f "):
refs = [parse_vertex_ref(token) for token in raw.split()[1:]]
for idx in range(1, len(refs) - 1):
faces.append((refs[0], refs[idx], refs[idx + 1]))
return np.asarray(vertices, dtype=np.float64), np.asarray(faces, dtype=np.int64)
def parse_vertex_ref(token: str) -> int:
return int(token.split("/")[0]) - 1
def triangle_normals(points: np.ndarray) -> np.ndarray:
normals = np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0])
norms = np.linalg.norm(normals, axis=1, keepdims=True)
return normals / np.maximum(norms, 1e-9)
def triangle_areas(points: np.ndarray) -> np.ndarray:
return np.linalg.norm(np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]), axis=1) * 0.5
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,910 @@
#!/usr/bin/env python3
"""One-image Real2Sim scene reconstruction workflow for MotrixSim.
This script is intentionally orchestration-only: it does not store API keys and
does not replace the project CLI. It chains the existing image-edit multiview
adapter, scene generation CLI, package verification, preview projection, and
archive creation into one reproducible command.
"""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import tarfile
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Any
import numpy as np
from PIL import Image, ImageDraw
PIL_BILINEAR = getattr(getattr(Image, "Resampling", Image), "BILINEAR")
REPO_ROOT = Path(__file__).resolve().parents[1]
if REPO_ROOT.as_posix() not in sys.path:
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.camera import load_anysplat_source_camera
DEFAULT_PROMPT = (
"Generate a reconstruction-grade, geometry-consistent 8-view image-to-image contact sheet for Real2Sim and MotrixSim import. "
"Use the input photo as exact source truth. Preserve all visible object identities, relative spacing, material colors, reflections, "
"shadows, the supporting surface, static background, camera scale, and lighting. Output exactly one 4 columns x 2 rows contact sheet, "
"8 equal portrait panels, thin white gutters, no labels, no captions, no extra text, no watermarks. Views in order: original front, "
"front-left 45 degrees, left 90 degrees, rear-left 135 degrees, rear/back 180 degrees, rear-right 225 degrees, right 270 degrees, "
"top oblique 35 degrees. Do not add people, hands, robots, extra objects, UI, a different room, or a product render background. "
"Prefer sharp contours and reconstruction-useful texture over artistic beautification."
)
def main() -> int:
parser = argparse.ArgumentParser(description="Generate and verify a MotrixSim-importable scene package from one image")
parser.add_argument("--image", type=Path, required=True, help="Source RGB image")
parser.add_argument("--output", type=Path, required=True, help="Output Real2Sim scene package directory")
parser.add_argument("--work-dir", type=Path, required=True, help="Intermediate workflow directory")
parser.add_argument("--prompt", default=DEFAULT_PROMPT, help="Image-to-image multiview prompt")
parser.add_argument("--scene-prompt", default="", help="Scene reconstruction prompt; defaults to --prompt")
parser.add_argument("--config", type=Path, default=REPO_ROOT / "configs/real2sim.external.example.json")
parser.add_argument("--backend", default="builtin", choices=["builtin", "external", "hybrid", "auto", "anysplat-direct"])
parser.add_argument("--api-key-env", default="REAL2SIM_IMAGE_EDIT_API_KEY")
parser.add_argument("--api-url", default=os.environ.get("REAL2SIM_IMAGE_EDIT_API_URL", "https://chatapi.applyma.com/v1/images/edits"))
parser.add_argument("--model", default=os.environ.get("REAL2SIM_IMAGE_EDIT_MODEL", "gpt-image-2"))
parser.add_argument("--size", default=os.environ.get("REAL2SIM_IMAGE_EDIT_SIZE", "1536x1024"))
parser.add_argument("--quality", default=os.environ.get("REAL2SIM_IMAGE_EDIT_QUALITY", "auto"))
parser.add_argument(
"--candidates",
type=int,
default=int(os.environ.get("REAL2SIM_IMAGE_EDIT_CANDIDATES", "1")),
help="Generate and score multiple image-edit multiview candidates, then reconstruct the best one",
)
parser.add_argument(
"--min-quality-score",
type=float,
default=float(os.environ.get("REAL2SIM_MIN_MULTIVIEW_QUALITY", "0.42")),
help="Fail before reconstruction when synthetic multiview quality is below this score",
)
parser.add_argument(
"--allow-low-quality",
action="store_true",
help="Continue even when the best synthetic multiview candidate is below --min-quality-score",
)
parser.add_argument("--frames-dir", type=Path, default=None, help="Reuse existing multiview frames instead of calling the image API")
parser.add_argument("--no-image-edit", action="store_true", help="Require --frames-dir and skip the image-edit API call")
parser.add_argument("--no-prune", action="store_true", help="Pass --no-prune to scene generation")
parser.add_argument("--anysplat-python", default=os.environ.get("REAL2SIM_ANY_SPLAT_PYTHON", ""), help="Python executable for AnySplat direct mode")
parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT"), help="AnySplat repository root for direct mode")
parser.add_argument("--anysplat-max-views", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MAX_VIEWS", "8")))
parser.add_argument("--anysplat-mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MESH_GRID", "48")))
parser.add_argument("--source-gs-preview", type=Path, default=None, help="Optional source-view 3DGS preview PNG to include in the Motrix package")
parser.add_argument(
"--min-source-preview-psnr",
type=float,
default=float(os.environ.get("REAL2SIM_MIN_SOURCE_PREVIEW_PSNR", "18.0")),
help="Fail AnySplat direct mode when source-view render PSNR is below this threshold",
)
parser.add_argument(
"--max-source-preview-mae",
type=float,
default=float(os.environ.get("REAL2SIM_MAX_SOURCE_PREVIEW_MAE", "0.10")),
help="Fail AnySplat direct mode when source-view render MAE is above this threshold",
)
parser.add_argument(
"--min-scene-vertices",
type=int,
default=int(os.environ.get("REAL2SIM_MIN_SCENE_VERTICES", "100000")),
help="Fail final package verification when scene.ply has fewer vertices",
)
parser.add_argument(
"--skip-final-package-verify",
action="store_true",
help="Skip final Motrix package archive verification",
)
parser.add_argument("--archive", type=Path, default=None, help="Optional .tar.gz path; defaults to <output>.tar.gz")
args = parser.parse_args()
if not args.image.is_file():
raise SystemExit(f"source image not found: {args.image}")
args.work_dir.mkdir(parents=True, exist_ok=True)
frames_dir = args.frames_dir
multiview_dir = args.work_dir / "synthetic_multiview"
quality_report = args.work_dir / "synthetic_multiview_quality.json"
if args.no_image_edit:
if frames_dir is None:
raise SystemExit("--no-image-edit requires --frames-dir")
elif frames_dir is None:
frames_dir = run_best_image_edit_candidate(args, multiview_dir, quality_report)
if frames_dir is None or not frames_dir.is_dir():
raise SystemExit(f"frames directory not found: {frames_dir}")
if not quality_report.exists():
write_multiview_quality_report(args.image, frames_dir, quality_report)
anysplat_metadata = None
if args.backend == "anysplat-direct":
anysplat_metadata = run_anysplat_direct_generation(args, frames_dir)
else:
run_scene_generation(args, frames_dir)
report = verify_motrix_package(args.output)
preview = write_projection_preview(args.output / "3dgs" / "scene.ply", args.output / "pointcloud_projection_preview.png")
if args.backend == "anysplat-direct" and (args.output / "motrix_import.json").is_file():
import_files = {
"manifest": args.output / "motrix_import.json",
"readme": args.output / "MOTRIX_IMPORT.md",
}
copy_quality_report(args.output, quality_report)
if anysplat_metadata is not None:
attach_anysplat_metadata_to_package(args.output, anysplat_metadata)
else:
import_files = write_motrix_import_files(args.output, report, preview, quality_report)
archive = create_archive(args.output, args.archive or args.output.with_suffix(".tar.gz"))
final_verification = None
if not args.skip_final_package_verify and args.backend == "anysplat-direct":
final_verification = run_final_package_verification(args, archive)
archive = create_archive(args.output, archive)
print(
json.dumps(
{
"package": report,
"preview": preview.as_posix(),
"motrix_import": import_files["manifest"].as_posix(),
"motrix_readme": import_files["readme"].as_posix(),
"archive": archive.as_posix(),
"anysplat_metadata": (args.output / "artifacts" / "anysplat_metadata.json").as_posix()
if (args.output / "artifacts" / "anysplat_metadata.json").is_file()
else None,
"final_package_verification": final_verification.as_posix() if final_verification is not None else None,
},
indent=2,
ensure_ascii=False,
)
)
return 0
def run_final_package_verification(args: argparse.Namespace, archive: Path) -> Path:
report_path = args.output / "motrix_package_verification.json"
command = [
sys.executable,
"scripts/real2sim_verify_motrix_package.py",
"--package",
archive.as_posix(),
"--min-scene-vertices",
str(args.min_scene_vertices),
"--min-source-preview-psnr",
str(args.min_source_preview_psnr),
"--max-source-preview-mae",
str(args.max_source_preview_mae),
]
completed = subprocess.run(command, cwd=REPO_ROOT, text=True, capture_output=True)
if completed.returncode != 0:
if completed.stdout:
print(completed.stdout, file=sys.stderr)
if completed.stderr:
print(completed.stderr, file=sys.stderr)
raise SystemExit(completed.returncode)
report_path.write_text(completed.stdout, encoding="utf-8")
return report_path
def run_best_image_edit_candidate(args: argparse.Namespace, multiview_dir: Path, quality_report: Path) -> Path:
if args.candidates <= 0:
raise SystemExit("--candidates must be positive")
candidate_reports: list[dict[str, Any]] = []
best_report: dict[str, Any] | None = None
best_frames: Path | None = None
for index in range(args.candidates):
suffix = f"candidate_{index + 1:02d}" if args.candidates > 1 else "candidate_01"
candidate_dir = multiview_dir / suffix
frames_dir = candidate_dir / "frames"
run_image_edit(args, candidate_dir, frames_dir)
report = score_multiview_frames(args.image, frames_dir)
report["candidate"] = index + 1
report["frames_dir"] = frames_dir.as_posix()
report["contact_sheet"] = (candidate_dir / "contact_sheet.png").as_posix()
candidate_reports.append(report)
if best_report is None or float(report["score"]) > float(best_report["score"]):
best_report = report
best_frames = frames_dir
if best_report is None or best_frames is None:
raise SystemExit("image-edit API produced no synthetic multiview candidates")
quality = {
"selected_candidate": best_report["candidate"],
"selected_frames_dir": best_frames.as_posix(),
"min_quality_score": args.min_quality_score,
"passed": float(best_report["score"]) >= args.min_quality_score,
"candidates": candidate_reports,
}
quality_report.parent.mkdir(parents=True, exist_ok=True)
quality_report.write_text(json.dumps(quality, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
if not quality["passed"] and not args.allow_low_quality:
raise SystemExit(
"synthetic multiview quality gate failed: "
f"best score {float(best_report['score']):.3f} < {args.min_quality_score:.3f}; "
"inspect synthetic_multiview_quality.json or rerun with --allow-low-quality"
)
return best_frames
def run_image_edit(args: argparse.Namespace, multiview_dir: Path, frames_dir: Path) -> None:
api_key = os.environ.get(args.api_key_env)
if not api_key:
raise SystemExit(f"missing image-edit API key: set {args.api_key_env}")
command = [
sys.executable,
"scripts/real2sim_external/synthetic_multiview_expand.py",
"--image",
args.image.as_posix(),
"--output-dir",
multiview_dir.as_posix(),
"--frames-dir",
frames_dir.as_posix(),
"--mode",
"scene",
"--api-url",
args.api_url,
"--model",
args.model,
"--size",
args.size,
"--quality",
args.quality,
"--prompt",
args.prompt,
]
env = os.environ.copy()
env["REAL2SIM_IMAGE_EDIT_API_KEY"] = api_key
run(command, env=env)
def write_multiview_quality_report(source_image: Path, frames_dir: Path, output: Path) -> None:
report = score_multiview_frames(source_image, frames_dir)
quality = {
"selected_candidate": None,
"selected_frames_dir": frames_dir.as_posix(),
"min_quality_score": None,
"passed": True,
"candidates": [report],
}
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(quality, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def score_multiview_frames(source_image: Path, frames_dir: Path) -> dict[str, Any]:
frame_paths = sorted(
path
for path in frames_dir.iterdir()
if path.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
)
frames = [Image.open(path).convert("RGB") for path in frame_paths]
source = Image.open(source_image).convert("RGB")
frame_count_score = min(len(frames), 8) / 8.0
resolution_score = 0.0
exposure_score = 0.0
sharpness_score = 0.0
if frames:
resolution_score = float(np.mean([min((img.width * img.height) / (360 * 360), 1.0) for img in frames]))
exposure_score = float(np.mean([exposure_quality(img) for img in frames]))
sharpness_score = float(np.mean([sharpness_quality(img) for img in frames]))
source_similarity = first_view_similarity(source, frames[0]) if frames else 0.0
diversity_score = view_diversity(frames)
score = (
0.20 * frame_count_score
+ 0.12 * resolution_score
+ 0.18 * source_similarity
+ 0.22 * diversity_score
+ 0.18 * sharpness_score
+ 0.10 * exposure_score
)
return {
"score": round(float(score), 4),
"frame_count": len(frames),
"frame_count_score": round(float(frame_count_score), 4),
"resolution_score": round(float(resolution_score), 4),
"source_similarity": round(float(source_similarity), 4),
"view_diversity": round(float(diversity_score), 4),
"sharpness_score": round(float(sharpness_score), 4),
"exposure_score": round(float(exposure_score), 4),
"frames": [path.as_posix() for path in frame_paths],
"warnings": multiview_quality_warnings(
frame_count=len(frames),
source_similarity=source_similarity,
diversity_score=diversity_score,
sharpness_score=sharpness_score,
exposure_score=exposure_score,
),
}
def image_array(image: Image.Image, size: tuple[int, int] = (160, 120)) -> np.ndarray:
return np.asarray(image.resize(size, PIL_BILINEAR), dtype=np.float32) / 255.0
def first_view_similarity(source: Image.Image, first_view: Image.Image) -> float:
src = image_array(source)
view = image_array(first_view)
mean_abs = float(np.mean(np.abs(src - view)))
color_delta = float(np.mean(np.abs(src.mean(axis=(0, 1)) - view.mean(axis=(0, 1)))))
return max(0.0, min(1.0, 1.0 - (1.7 * mean_abs + 0.8 * color_delta)))
def view_diversity(frames: list[Image.Image]) -> float:
if len(frames) < 2:
return 0.0
arrays = [image_array(frame, size=(96, 72)) for frame in frames]
diffs = [float(np.mean(np.abs(arrays[index] - arrays[index - 1]))) for index in range(1, len(arrays))]
mean_diff = float(np.mean(diffs))
if mean_diff < 0.035:
return mean_diff / 0.035 * 0.45
if mean_diff > 0.42:
return max(0.0, 1.0 - (mean_diff - 0.42) / 0.30)
return min(1.0, 0.45 + (mean_diff - 0.035) / (0.42 - 0.035) * 0.55)
def sharpness_quality(image: Image.Image) -> float:
gray = np.asarray(image.convert("L").resize((192, 144), PIL_BILINEAR), dtype=np.float32) / 255.0
gx = np.diff(gray, axis=1)
gy = np.diff(gray, axis=0)
energy = float(np.mean(np.abs(gx)) + np.mean(np.abs(gy)))
return max(0.0, min(1.0, energy / 0.11))
def exposure_quality(image: Image.Image) -> float:
gray = np.asarray(image.convert("L").resize((96, 72), PIL_BILINEAR), dtype=np.float32) / 255.0
clipped = float(np.mean((gray < 0.025) | (gray > 0.975)))
contrast = float(np.std(gray))
return max(0.0, min(1.0, (1.0 - clipped * 2.0) * min(contrast / 0.22, 1.0)))
def multiview_quality_warnings(
*,
frame_count: int,
source_similarity: float,
diversity_score: float,
sharpness_score: float,
exposure_score: float,
) -> list[str]:
warnings: list[str] = []
if frame_count < 8:
warnings.append("expected 8 synthetic views")
if source_similarity < 0.35:
warnings.append("first synthetic view may not preserve the source image")
if diversity_score < 0.35:
warnings.append("synthetic views may be too similar for scene reconstruction")
if sharpness_score < 0.35:
warnings.append("synthetic views look blurry or low-detail")
if exposure_score < 0.35:
warnings.append("synthetic views have weak contrast or clipped exposure")
return warnings
def run_scene_generation(args: argparse.Namespace, frames_dir: Path) -> None:
command = [
sys.executable,
"-m",
"real2sim",
"generate-scene",
"--input",
frames_dir.as_posix(),
"--output",
args.output.as_posix(),
"--work-dir",
(args.work_dir / "scene_work").as_posix(),
"--backend",
args.backend,
"--config",
args.config.as_posix(),
"--prompt",
args.scene_prompt or args.prompt,
]
if args.no_prune:
command.append("--no-prune")
run(command)
def run_anysplat_direct_generation(args: argparse.Namespace, frames_dir: Path) -> Path | None:
scene_dir = args.work_dir / "anysplat_scene"
scene_dir.mkdir(parents=True, exist_ok=True)
scene_ply = scene_dir / "scene.ply"
scene_mesh = scene_dir / "mesh.obj"
cameras_json = scene_dir / "cameras.json"
depth_map = scene_dir / "depth.pgm"
source_preview = args.source_gs_preview or (scene_dir / "source_gs_preview.png")
anysplat_python = args.anysplat_python or sys.executable
command = [
anysplat_python,
"scripts/real2sim_external/anysplat_scene.py",
"--image",
args.image.as_posix(),
"--frames-dir",
frames_dir.as_posix(),
"--output-dir",
scene_dir.as_posix(),
"--ply",
scene_ply.as_posix(),
"--mesh",
scene_mesh.as_posix(),
"--cameras",
cameras_json.as_posix(),
"--depth",
depth_map.as_posix(),
"--source-preview",
source_preview.as_posix(),
"--max-views",
str(args.anysplat_max_views),
"--mesh-grid",
str(max(8, int(getattr(args, "anysplat_mesh_grid", 48)))),
]
if args.anysplat_root is not None:
command.extend(["--anysplat-root", args.anysplat_root.as_posix()])
run(command)
metadata = scene_dir / "anysplat_metadata.json"
if metadata.exists():
enforce_anysplat_preview_quality(
metadata,
min_psnr=args.min_source_preview_psnr,
max_mae=args.max_source_preview_mae,
allow_low_quality=args.allow_low_quality,
)
package_command = [
sys.executable,
"-m",
"real2sim",
"package-existing",
"--source",
args.image.as_posix(),
"--scene-ply",
scene_ply.as_posix(),
"--scene-mesh",
scene_mesh.as_posix(),
"--cameras-json",
cameras_json.as_posix(),
"--depth-map",
depth_map.as_posix(),
"--background-image",
args.image.as_posix(),
"--output",
args.output.as_posix(),
"--prompt",
args.scene_prompt or args.prompt,
]
package_command.extend(["--preview-image", source_preview.as_posix()])
run(package_command)
return metadata if metadata.exists() else None
def copy_quality_report(package_dir: Path, quality_report: Path) -> None:
if quality_report.exists():
quality_dest = package_dir / "synthetic_multiview_quality.json"
quality_dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8")
def enforce_anysplat_preview_quality(
metadata_path: Path,
*,
min_psnr: float,
max_mae: float,
allow_low_quality: bool,
) -> dict[str, Any]:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
metrics = metadata.get("source_preview_metrics")
if not isinstance(metrics, dict):
raise SystemExit(f"AnySplat metadata missing source_preview_metrics: {metadata_path}")
psnr = float(metrics.get("psnr", 0.0))
mae = float(metrics.get("mae", 1.0))
passed = psnr >= min_psnr and mae <= max_mae
gate = {
"min_source_preview_psnr": min_psnr,
"max_source_preview_mae": max_mae,
"passed": passed,
}
metadata["source_preview_quality_gate"] = gate
metadata_path.write_text(json.dumps(metadata, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
if not passed and not allow_low_quality:
raise SystemExit(
"AnySplat source-view quality gate failed: "
f"psnr={psnr:.3f} < {min_psnr:.3f} or mae={mae:.4f} > {max_mae:.4f}; "
"rerun with --allow-low-quality to keep the package"
)
return metadata
def attach_anysplat_metadata_to_package(package_dir: Path, metadata_path: Path) -> Path:
artifact_dir = package_dir / "artifacts"
artifact_dir.mkdir(parents=True, exist_ok=True)
dest = artifact_dir / "anysplat_metadata.json"
dest.write_text(metadata_path.read_text(encoding="utf-8"), encoding="utf-8")
import_path = package_dir / "motrix_import.json"
if import_path.is_file():
import_manifest = json.loads(import_path.read_text(encoding="utf-8"))
import_manifest["reconstruction_metadata"] = dest.relative_to(package_dir).as_posix()
metadata = json.loads(dest.read_text(encoding="utf-8"))
metrics = metadata.get("source_preview_metrics")
gate = metadata.get("source_preview_quality_gate")
if isinstance(metrics, dict):
import_manifest.setdefault("verification", {})["source_preview_metrics"] = metrics
if isinstance(gate, dict):
import_manifest.setdefault("verification", {})["source_preview_quality_gate"] = gate
import_path.write_text(json.dumps(import_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
return dest
def verify_motrix_package(package_dir: Path) -> dict[str, Any]:
scene_xml = package_dir / "mjcf" / "scene.xml"
scene_ply = package_dir / "3dgs" / "scene.ply"
config_json = package_dir / "configs" / "go2_real2sim.json"
manifest_json = package_dir / "package_manifest.json"
cameras_json = package_dir / "artifacts" / "cameras_json.json"
for path in (scene_xml, scene_ply, config_json, manifest_json, cameras_json):
if not path.is_file() or path.stat().st_size <= 0:
raise SystemExit(f"package verification failed; missing or empty file: {path}")
ET.parse(scene_xml)
config = json.loads(config_json.read_text(encoding="utf-8"))
manifest = json.loads(manifest_json.read_text(encoding="utf-8"))
cameras = json.loads(cameras_json.read_text(encoding="utf-8"))
load_anysplat_source_camera(cameras_json)
scene_gaussians = config.get("scene_gaussians")
if not isinstance(scene_gaussians, dict) or "scene" not in scene_gaussians:
raise SystemExit("package verification failed; config missing scene_gaussians.scene")
gaussian_path = resolve_package_config_path(config_json, scene_gaussians["scene"])
if not gaussian_path.is_file():
raise SystemExit(f"package verification failed; scene Gaussian PLY does not exist: {gaussian_path}")
manifest_scene = manifest.get("scene", {})
manifest_ply = package_dir / str(manifest_scene.get("gaussian_ply", ""))
if not manifest_ply.is_file():
raise SystemExit(f"package verification failed; manifest scene gaussian_ply does not exist: {manifest_ply}")
return {
"root": package_dir.as_posix(),
"scene_xml": scene_xml.as_posix(),
"config_json": config_json.as_posix(),
"scene_ply": scene_ply.as_posix(),
"scene_ply_vertices": count_ply_vertices(scene_ply),
"capture_kind": manifest.get("metadata", {}).get("capture_kind"),
"scene_fusion": manifest.get("metadata", {}).get("builtin_scene_fusion"),
"scene_view_count": manifest.get("metadata", {}).get("builtin_scene_view_count"),
"camera_view_count": len(cameras.get("views", [])) if isinstance(cameras.get("views"), list) else 0,
}
def resolve_package_config_path(config_json: Path, value: Any) -> Path:
path = Path(str(value))
if path.is_absolute():
return path
return (config_json.parent / path).resolve()
def count_ply_vertices(path: Path) -> int:
with path.open("r", encoding="ascii", errors="ignore") as handle:
for line in handle:
if line.startswith("element vertex "):
return int(line.split()[-1])
if line.strip() == "end_header":
break
raise SystemExit(f"PLY vertex count not found: {path}")
def write_projection_preview(ply: Path, output: Path) -> Path:
points, colors = load_ply_points_for_preview(ply, max_points=90000)
width, height = 900, 520
canvas = Image.new("RGB", (width * 2 + 20, height), (245, 245, 245))
draw_projection(canvas, points, colors, (0, 2), 0, width, height, "front projection: x/z")
draw_projection(canvas, points, colors, (0, 1), width + 20, width, height, "top projection: x/y")
output.parent.mkdir(parents=True, exist_ok=True)
canvas.save(output)
return output
def write_motrix_import_files(
package_dir: Path,
report: dict[str, Any],
preview: Path,
quality_report: Path | None = None,
) -> dict[str, Path]:
"""Write explicit MotrixSim import entrypoints into the generated package."""
manifest_path = package_dir / "motrix_import.json"
readme_path = package_dir / "MOTRIX_IMPORT.md"
package_name = package_dir.name
recommended_camera = "source_3dgs_preview"
viewer_args = [
"--package",
f"/path/to/{package_name}",
"--camera",
"closeup",
"--gs-width",
"900",
"--gs-height",
"700",
"--static-gs-preview",
"--with-gs",
"--show-source-gs",
"--source-gs-only",
]
import_manifest = {
"format": "gs-playground-real2sim-motrix-import-v1",
"import_type": "mjcf_scene_with_3dgs",
"scene_xml": "mjcf/scene.xml",
"config": "configs/go2_real2sim.json",
"scene_gaussians": {"scene": "3dgs/scene.ply"},
"camera_artifacts": {"cameras_json": "artifacts/cameras_json.json"},
"preview": preview.relative_to(package_dir).as_posix(),
"recommended_camera": recommended_camera,
"viewer_args": viewer_args,
"wuying_archive_launch": (
"scripts/real2sim_wuying_launch_package.sh "
f"--archive /path/to/{package_name}.tar.gz "
f"--dest /tmp/{package_name}_import --with-gs"
),
"verification": {
"scene_ply_vertices": report.get("scene_ply_vertices"),
"scene_fusion": report.get("scene_fusion"),
"scene_view_count": report.get("scene_view_count"),
"camera_view_count": report.get("camera_view_count"),
"recommended_preview": recommended_camera,
},
}
if quality_report is not None and quality_report.exists():
quality_dest = package_dir / "synthetic_multiview_quality.json"
quality_dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8")
import_manifest["synthetic_multiview_quality"] = quality_dest.relative_to(package_dir).as_posix()
manifest_path.write_text(json.dumps(import_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
readme_path.write_text(
"\n".join(
[
"# MotrixSim Import",
"",
"This package is a scene-level Real2Sim reconstruction from one source image plus synthetic multiview completion.",
"",
"Required entrypoints:",
"",
"- MJCF scene: `mjcf/scene.xml`",
"- Scene Gaussian asset: `3dgs/scene.ply`",
"- Viewer config: `configs/go2_real2sim.json`",
"- Source cameras: `artifacts/cameras_json.json`",
"- Projection preview: `pointcloud_projection_preview.png`",
"",
"Wuying launch command:",
"",
"```bash",
import_manifest["wuying_archive_launch"],
"```",
"",
"Direct viewer arguments:",
"",
"```bash",
" ".join(import_manifest["viewer_args"]),
"```",
"",
"Verification:",
"",
f"- scene vertices: {report.get('scene_ply_vertices')}",
f"- scene fusion: {report.get('scene_fusion')}",
f"- scene views: {report.get('scene_view_count')}",
f"- camera views: {report.get('camera_view_count')}",
"",
]
),
encoding="utf-8",
)
return {"manifest": manifest_path, "readme": readme_path}
def load_ply_points_for_preview(path: Path, *, max_points: int) -> tuple[np.ndarray, np.ndarray]:
ply_format = read_ply_format(path)
if ply_format != "ascii":
return load_binary_ply_points_for_preview(path, max_points=max_points)
with path.open("r", encoding="ascii", errors="ignore") as handle:
vertex_count = 0
for line in handle:
if line.startswith("element vertex "):
vertex_count = int(line.split()[-1])
if line.strip() == "end_header":
break
stride = max(1, vertex_count // max_points)
points = []
colors = []
for index, line in enumerate(handle):
if index >= vertex_count:
break
if index % stride:
continue
values = line.split()
if len(values) < 9:
continue
points.append(tuple(map(float, values[:3])))
rgb = np.asarray(list(map(float, values[6:9])), dtype=np.float32)
rgb = 1.0 / (1.0 + np.exp(-rgb))
colors.append((rgb * 255.0).clip(0.0, 255.0).astype(np.uint8))
if not points:
raise SystemExit(f"no previewable points found in {path}")
return np.asarray(points, dtype=np.float32), np.asarray(colors, dtype=np.uint8)
def read_ply_format(path: Path) -> str:
with path.open("rb") as handle:
for raw in handle:
line = raw.decode("ascii", errors="ignore").strip()
if line.startswith("format "):
parts = line.split()
return parts[1] if len(parts) > 1 else "ascii"
if line == "end_header":
break
return "ascii"
def load_binary_ply_points_for_preview(path: Path, *, max_points: int) -> tuple[np.ndarray, np.ndarray]:
try:
from plyfile import PlyData
except Exception as exc:
return load_binary_ply_points_without_plyfile(path, max_points=max_points, import_error=exc)
ply = PlyData.read(path)
vertex = ply["vertex"].data
names = set(vertex.dtype.names or ())
required = {"x", "y", "z"}
if not required.issubset(names):
raise SystemExit(f"binary PLY missing required xyz fields: {path}")
vertex_count = len(vertex)
stride = max(1, vertex_count // max_points)
sampled = vertex[::stride]
points = np.column_stack([sampled["x"], sampled["y"], sampled["z"]]).astype(np.float32, copy=False)
if {"red", "green", "blue"}.issubset(names):
colors = np.column_stack([sampled["red"], sampled["green"], sampled["blue"]]).astype(np.uint8, copy=False)
elif {"f_dc_0", "f_dc_1", "f_dc_2"}.issubset(names):
rgb = np.column_stack([sampled["f_dc_0"], sampled["f_dc_1"], sampled["f_dc_2"]]).astype(np.float32, copy=False)
colors = (1.0 / (1.0 + np.exp(-rgb)) * 255.0).clip(0.0, 255.0).astype(np.uint8)
else:
colors = np.full((len(points), 3), 220, dtype=np.uint8)
if len(points) == 0:
raise SystemExit(f"no previewable points found in {path}")
return points, colors
def load_binary_ply_points_without_plyfile(
path: Path,
*,
max_points: int,
import_error: Exception | None = None,
) -> tuple[np.ndarray, np.ndarray]:
data = path.read_bytes()
header_end = data.find(b"end_header\n")
newline_len = 1
if header_end < 0:
header_end = data.find(b"end_header\r\n")
newline_len = 2
if header_end < 0:
raise SystemExit(f"binary PLY missing end_header: {path}") from import_error
header_text = data[: header_end + len("end_header")].decode("ascii", errors="ignore")
header_lines = [line.strip() for line in header_text.splitlines()]
ply_format = "binary_little_endian"
vertex_count = 0
vertex_properties: list[tuple[str, str]] = []
in_vertex = False
for line in header_lines:
if line.startswith("format "):
parts = line.split()
if len(parts) > 1:
ply_format = parts[1]
elif line.startswith("element "):
parts = line.split()
in_vertex = len(parts) >= 3 and parts[1] == "vertex"
if in_vertex:
vertex_count = int(parts[2])
elif in_vertex and line.startswith("property "):
parts = line.split()
if len(parts) == 3:
vertex_properties.append((parts[1], parts[2]))
elif len(parts) > 3 and parts[1] == "list":
raise SystemExit(f"binary PLY vertex list properties are unsupported for preview: {path}") from import_error
if vertex_count <= 0 or not vertex_properties:
raise SystemExit(f"binary PLY has no previewable vertex table: {path}") from import_error
endian = "<" if ply_format == "binary_little_endian" else ">"
dtype = np.dtype([(name, numpy_dtype_for_ply_type(kind, endian)) for kind, name in vertex_properties])
offset = header_end + len("end_header") + newline_len
vertex = np.frombuffer(data, dtype=dtype, count=vertex_count, offset=offset)
names = set(vertex.dtype.names or ())
if not {"x", "y", "z"}.issubset(names):
raise SystemExit(f"binary PLY missing required xyz fields: {path}") from import_error
stride = max(1, vertex_count // max_points)
sampled = vertex[::stride]
points = np.column_stack([sampled["x"], sampled["y"], sampled["z"]]).astype(np.float32, copy=False)
if {"red", "green", "blue"}.issubset(names):
colors = np.column_stack([sampled["red"], sampled["green"], sampled["blue"]]).astype(np.uint8, copy=False)
elif {"f_dc_0", "f_dc_1", "f_dc_2"}.issubset(names):
rgb = np.column_stack([sampled["f_dc_0"], sampled["f_dc_1"], sampled["f_dc_2"]]).astype(np.float32, copy=False)
colors = (1.0 / (1.0 + np.exp(-rgb)) * 255.0).clip(0.0, 255.0).astype(np.uint8)
else:
colors = np.full((len(points), 3), 220, dtype=np.uint8)
return points, colors
def numpy_dtype_for_ply_type(kind: str, endian: str) -> str:
mapping = {
"char": "i1",
"int8": "i1",
"uchar": "u1",
"uint8": "u1",
"short": "i2",
"int16": "i2",
"ushort": "u2",
"uint16": "u2",
"int": "i4",
"int32": "i4",
"uint": "u4",
"uint32": "u4",
"float": "f4",
"float32": "f4",
"double": "f8",
"float64": "f8",
}
if kind not in mapping:
raise SystemExit(f"unsupported binary PLY property type: {kind}")
code = mapping[kind]
return code if code.endswith("1") else endian + code
def draw_projection(
canvas: Image.Image,
points: np.ndarray,
colors: np.ndarray,
axes: tuple[int, int],
offset: int,
width: int,
height: int,
title: str,
) -> None:
sub = Image.new("RGB", (width, height), (250, 250, 250))
draw = ImageDraw.Draw(sub)
xy = points[:, list(axes)]
low = xy.min(axis=0)
high = xy.max(axis=0)
span = np.maximum(high - low, 1e-6)
uv = (xy - low) / span
px = (uv[:, 0] * (width - 60) + 30).astype(int)
py = ((1.0 - uv[:, 1]) * (height - 60) + 30).astype(int)
order = np.argsort(points[:, 1] if axes != (0, 1) else points[:, 2])
for index in order:
x = int(px[index])
y = int(py[index])
if 0 <= x < width and 0 <= y < height:
sub.putpixel((x, y), tuple(int(v) for v in colors[index]))
draw.rectangle((0, 0, width - 1, height - 1), outline=(190, 190, 190))
draw.text((14, 12), title, fill=(30, 30, 30))
canvas.paste(sub, (offset, 0))
def create_archive(package_dir: Path, archive: Path) -> Path:
archive.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(archive, "w:gz") as tar:
tar.add(package_dir, arcname=package_dir.name)
return archive
def run(command: list[str], *, env: dict[str, str] | None = None) -> None:
result = subprocess.run(command, cwd=REPO_ROOT, env=env, text=True)
if result.returncode != 0:
raise SystemExit(result.returncode)
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,381 @@
#!/usr/bin/env python3
"""Convert a colored PLY mesh to a Motrix-friendly OBJ.
The converter keeps vertex colors for tools that understand them and also emits
an OBJ material palette so viewers that ignore OBJ vertex colors still show the
scene with explicit diffuse materials.
"""
from __future__ import annotations
import argparse
import json
import math
import re
import struct
from dataclasses import dataclass
from pathlib import Path
from typing import Any, BinaryIO
SCALAR_TYPES: dict[str, tuple[str, int]] = {
"char": ("b", 1),
"int8": ("b", 1),
"uchar": ("B", 1),
"uint8": ("B", 1),
"short": ("h", 2),
"int16": ("h", 2),
"ushort": ("H", 2),
"uint16": ("H", 2),
"int": ("i", 4),
"int32": ("i", 4),
"uint": ("I", 4),
"uint32": ("I", 4),
"float": ("f", 4),
"float32": ("f", 4),
"double": ("d", 8),
"float64": ("d", 8),
}
COLOR_NAMES = (
("red", "green", "blue"),
("diffuse_red", "diffuse_green", "diffuse_blue"),
("r", "g", "b"),
)
@dataclass(frozen=True)
class PlyHeader:
ply_format: str
vertex_count: int
face_count: int
vertex_properties: tuple[tuple[str, str], ...]
face_count_type: str
face_index_type: str
data_offset: int
@dataclass(frozen=True)
class ObjVertex:
x: float
y: float
z: float
r: float
g: float
b: float
@dataclass(frozen=True)
class ObjTriangle:
a: int
b: int
c: int
def main() -> int:
parser = argparse.ArgumentParser(description="Convert a colored PLY mesh to a Motrix-compatible OBJ")
parser.add_argument("--input-ply", type=Path, required=True)
parser.add_argument("--output-obj", type=Path, required=True)
parser.add_argument("--report", type=Path, default=None)
parser.add_argument("--no-material-palette", action="store_true", help="Write vertex-color-only OBJ without a sidecar MTL")
parser.add_argument("--material-palette-levels", type=int, default=18, help="RGB quantization levels for generated diffuse materials")
args = parser.parse_args()
report = convert_ply_mesh_to_obj(
args.input_ply,
args.output_obj,
material_palette=not args.no_material_palette,
material_palette_levels=max(int(args.material_palette_levels), 2),
)
if args.report is not None:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
else:
print(json.dumps(report, indent=2, ensure_ascii=False))
return 0
def convert_ply_mesh_to_obj(
input_ply: Path,
output_obj: Path,
*,
material_palette: bool = True,
material_palette_levels: int = 18,
) -> dict[str, Any]:
header = read_ply_header(input_ply)
output_obj.parent.mkdir(parents=True, exist_ok=True)
with input_ply.open("rb") as src:
src.seek(header.data_offset)
if header.ply_format == "ascii":
vertices, triangles = read_ascii_mesh(src, header)
elif header.ply_format == "binary_little_endian":
vertices, triangles = read_binary_mesh(src, header, endian="<")
elif header.ply_format == "binary_big_endian":
vertices, triangles = read_binary_mesh(src, header, endian=">")
else:
raise SystemExit(f"unsupported PLY format: {header.ply_format}")
material_count = write_obj_mesh(
input_ply=input_ply,
output_obj=output_obj,
vertices=vertices,
triangles=triangles,
material_palette=material_palette,
material_palette_levels=max(int(material_palette_levels), 2),
)
return {
"format": "gs-playground-real2sim-ply-mesh-to-obj-report-v1",
"input_ply": input_ply.resolve().as_posix(),
"output_obj": output_obj.resolve().as_posix(),
"vertex_count": int(header.vertex_count),
"face_count": int(len(triangles)),
"texcoord_count": int(len(vertices)),
"material_palette": bool(material_palette),
"material_palette_levels": int(max(int(material_palette_levels), 2)),
"material_count": int(material_count),
"material_library": output_obj.with_suffix(".mtl").resolve().as_posix() if material_palette else None,
}
def read_ply_header(path: Path) -> PlyHeader:
ply_format = "ascii"
vertex_count = 0
face_count = 0
vertex_properties: list[tuple[str, str]] = []
face_count_type = "uchar"
face_index_type = "int"
active_element: str | None = None
with path.open("rb") as handle:
first = handle.readline().decode("ascii", errors="ignore").strip()
if first != "ply":
raise SystemExit(f"not a PLY file: {path}")
while True:
raw = handle.readline()
if not raw:
raise SystemExit(f"PLY header missing end_header: {path}")
line = raw.decode("ascii", errors="ignore").strip()
if line == "end_header":
data_offset = handle.tell()
break
parts = line.split()
if not parts:
continue
if parts[0] == "format" and len(parts) >= 2:
ply_format = parts[1]
elif parts[0] == "element" and len(parts) >= 3:
active_element = parts[1]
if active_element == "vertex":
vertex_count = int(parts[2])
elif active_element == "face":
face_count = int(parts[2])
elif parts[0] == "property" and active_element == "vertex" and len(parts) == 3:
vertex_properties.append((parts[2], parts[1]))
elif parts[0] == "property" and active_element == "face" and len(parts) >= 5 and parts[1] == "list":
face_count_type = parts[2]
face_index_type = parts[3]
if vertex_count <= 0:
raise SystemExit(f"PLY has no vertices: {path}")
return PlyHeader(
ply_format=ply_format,
vertex_count=vertex_count,
face_count=face_count,
vertex_properties=tuple(vertex_properties),
face_count_type=face_count_type,
face_index_type=face_index_type,
data_offset=data_offset,
)
def read_ascii_mesh(src: BinaryIO, header: PlyHeader) -> tuple[list[ObjVertex], list[ObjTriangle]]:
property_names = [name for name, _kind in header.vertex_properties]
index = {name: idx for idx, name in enumerate(property_names)}
color_indices = find_color_indices(index)
vertices: list[ObjVertex] = []
for _ in range(header.vertex_count):
parts = src.readline().decode("utf-8", errors="ignore").split()
if len(parts) < len(property_names):
raise SystemExit("truncated ASCII PLY vertex table")
vertices.append(make_obj_vertex(parts, index, color_indices))
triangles: list[ObjTriangle] = []
for _ in range(header.face_count):
parts = src.readline().decode("utf-8", errors="ignore").split()
if not parts:
continue
count = int(parts[0])
indices = [int(value) + 1 for value in parts[1 : 1 + count]]
triangles.extend(triangulate(indices))
return vertices, triangles
def read_binary_mesh(src: BinaryIO, header: PlyHeader, *, endian: str) -> tuple[list[ObjVertex], list[ObjTriangle]]:
property_names = [name for name, _kind in header.vertex_properties]
index = {name: idx for idx, name in enumerate(property_names)}
color_indices = find_color_indices(index)
vertex_struct = struct.Struct(endian + "".join(scalar_format(kind) for _name, kind in header.vertex_properties))
vertices: list[ObjVertex] = []
for _ in range(header.vertex_count):
data = src.read(vertex_struct.size)
if len(data) != vertex_struct.size:
raise SystemExit("truncated binary PLY vertex table")
values = vertex_struct.unpack(data)
vertices.append(make_obj_vertex(values, index, color_indices))
count_struct = struct.Struct(endian + scalar_format(header.face_count_type))
index_struct = struct.Struct(endian + scalar_format(header.face_index_type))
triangles: list[ObjTriangle] = []
for _ in range(header.face_count):
raw_count = src.read(count_struct.size)
if len(raw_count) != count_struct.size:
raise SystemExit("truncated binary PLY face table")
count = int(count_struct.unpack(raw_count)[0])
raw_indices = src.read(index_struct.size * count)
if len(raw_indices) != index_struct.size * count:
raise SystemExit("truncated binary PLY face indices")
indices = [
int(index_struct.unpack_from(raw_indices, offset)[0]) + 1
for offset in range(0, len(raw_indices), index_struct.size)
]
triangles.extend(triangulate(indices))
return vertices, triangles
def scalar_format(kind: str) -> str:
if kind not in SCALAR_TYPES:
raise SystemExit(f"unsupported PLY scalar type: {kind}")
return SCALAR_TYPES[kind][0]
def find_color_indices(index: dict[str, int]) -> tuple[int, int, int] | None:
for names in COLOR_NAMES:
if all(name in index for name in names):
return tuple(index[name] for name in names)
return None
def make_obj_vertex(values: Any, index: dict[str, int], color_indices: tuple[int, int, int] | None) -> ObjVertex:
x = float(values[index["x"]])
y = float(values[index["y"]])
z = float(values[index["z"]])
if color_indices is None:
r, g, b = 0.72, 0.72, 0.72
else:
channels = [float(values[idx]) for idx in color_indices]
if max(channels) > 1.0:
channels = [value / 255.0 for value in channels]
r, g, b = [min(max(value, 0.0), 1.0) for value in channels]
return ObjVertex(x=x, y=y, z=z, r=r, g=g, b=b)
def triangulate(indices: list[int]) -> list[ObjTriangle]:
if len(indices) < 3:
return []
triangles: list[ObjTriangle] = []
for idx in range(1, len(indices) - 1):
triangles.append(ObjTriangle(indices[0], indices[idx], indices[idx + 1]))
return triangles
def write_obj_mesh(
*,
input_ply: Path,
output_obj: Path,
vertices: list[ObjVertex],
triangles: list[ObjTriangle],
material_palette: bool,
material_palette_levels: int,
) -> int:
material_path = output_obj.with_suffix(".mtl")
uv_axes = choose_uv_axes(vertices)
uv_bounds = axis_bounds(vertices, uv_axes)
materials: dict[str, tuple[float, float, float]] = {}
with output_obj.open("w", encoding="utf-8") as dst:
dst.write(f"# Converted from {input_ply.name} for Real2Sim Motrix scene mesh\n")
if material_palette:
dst.write(f"mtllib {material_path.name}\n")
dst.write("o real2sim_colmap_dense_scene\n")
for vertex in vertices:
dst.write(
f"v {vertex.x:.9g} {vertex.y:.9g} {vertex.z:.9g} "
f"{vertex.r:.6f} {vertex.g:.6f} {vertex.b:.6f}\n"
)
for vertex in vertices:
u, v = vertex_uv(vertex, uv_axes, uv_bounds)
dst.write(f"vt {u:.7g} {v:.7g}\n")
current_material = None
for triangle in triangles:
if material_palette:
material_name, color = triangle_material(vertices, triangle, output_obj.stem, material_palette_levels)
materials.setdefault(material_name, color)
if material_name != current_material:
dst.write(f"usemtl {material_name}\n")
current_material = material_name
dst.write(f"f {triangle.a}/{triangle.a} {triangle.b}/{triangle.b} {triangle.c}/{triangle.c}\n")
if material_palette:
write_material_library(material_path, materials)
elif material_path.exists():
material_path.unlink()
return len(materials)
def choose_uv_axes(vertices: list[ObjVertex]) -> tuple[str, str]:
if not vertices:
return ("x", "y")
extents = {}
for axis in ("x", "y", "z"):
values = [getattr(vertex, axis) for vertex in vertices]
extents[axis] = max(values) - min(values)
ranked = sorted(extents, key=lambda axis: extents[axis], reverse=True)
return ranked[0], ranked[1]
def axis_bounds(vertices: list[ObjVertex], axes: tuple[str, str]) -> tuple[tuple[float, float], tuple[float, float]]:
bounds: list[tuple[float, float]] = []
for axis in axes:
values = [getattr(vertex, axis) for vertex in vertices]
low = min(values) if values else 0.0
high = max(values) if values else 1.0
if math.isclose(high, low):
high = low + 1.0
bounds.append((low, high))
return bounds[0], bounds[1]
def vertex_uv(vertex: ObjVertex, axes: tuple[str, str], bounds: tuple[tuple[float, float], tuple[float, float]]) -> tuple[float, float]:
first, second = axes
(u_low, u_high), (v_low, v_high) = bounds
u = (getattr(vertex, first) - u_low) / max(u_high - u_low, 1e-9)
v = (getattr(vertex, second) - v_low) / max(v_high - v_low, 1e-9)
return min(max(u, 0.0), 1.0), min(max(v, 0.0), 1.0)
def triangle_material(
vertices: list[ObjVertex],
triangle: ObjTriangle,
stem: str,
levels: int,
) -> tuple[str, tuple[float, float, float]]:
tri_vertices = [vertices[index - 1] for index in (triangle.a, triangle.b, triangle.c)]
color = tuple(sum(getattr(vertex, channel) for vertex in tri_vertices) / 3.0 for channel in ("r", "g", "b"))
quantized = tuple(round(channel * (levels - 1)) / (levels - 1) for channel in color)
suffix = "_".join(f"{int(round(channel * 255)):02x}" for channel in quantized)
return f"{safe_token(stem)}_mat_{suffix}", quantized
def safe_token(value: str) -> str:
token = re.sub(r"[^A-Za-z0-9_]+", "_", value.strip())
return token.strip("_") or "scene"
def write_material_library(path: Path, materials: dict[str, tuple[float, float, float]]) -> None:
with path.open("w", encoding="utf-8") as dst:
dst.write("# Generated Real2Sim dense scene material palette\n")
for name, (r, g, b) in sorted(materials.items()):
dst.write(f"newmtl {name}\n")
dst.write(f"Ka {r:.6f} {g:.6f} {b:.6f}\n")
dst.write(f"Kd {r:.6f} {g:.6f} {b:.6f}\n")
dst.write("Ks 0.030000 0.030000 0.030000\n")
dst.write("Ns 8.000000\n")
dst.write("d 1.000000\n")
dst.write("illum 2\n\n")
if __name__ == "__main__":
raise SystemExit(main())

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""Run SAM-3D over movable extraction tasks and emit production object assets."""
from __future__ import annotations
import argparse
import json
import os
import shlex
import subprocess
import sys
from pathlib import Path
from typing import Any
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from real2sim.contracts import write_json
from real2sim.image_io import read_gray, read_rgb, write_gray, write_rgb
def main() -> int:
parser = argparse.ArgumentParser(description="Convert movable extraction tasks into production SAM-3D object assets")
parser.add_argument("--package", type=Path, required=True, help="Real2Sim package root")
parser.add_argument("--tasks", type=Path, default=None, help="movable_extraction_tasks.json; defaults under package")
parser.add_argument("--output-objects-json", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--command", default=os.environ.get("REAL2SIM_SAM3D_COMMAND", ""))
parser.add_argument("--max-objects", type=int, default=3)
parser.add_argument("--dry-run", action="store_true", help="Write planned SAM-3D jobs without executing them")
args = parser.parse_args()
result = run_sam3d_movable_batch(
package_root=args.package,
tasks_path=args.tasks or args.package / "artifacts" / "assets" / "movable_extraction_tasks.json",
output_objects_json=args.output_objects_json,
output_dir=args.output_dir,
command_template=args.command,
max_objects=max(1, int(args.max_objects)),
dry_run=bool(args.dry_run),
)
print(json.dumps(result, indent=2, ensure_ascii=False))
return 0
def run_sam3d_movable_batch(
*,
package_root: Path,
tasks_path: Path,
output_objects_json: Path,
output_dir: Path,
command_template: str,
max_objects: int,
dry_run: bool = False,
) -> dict[str, Any]:
tasks_data = json.loads(tasks_path.read_text(encoding="utf-8"))
tasks = tasks_data.get("tasks", []) if isinstance(tasks_data, dict) else []
if not dry_run and not command_template.strip():
raise SystemExit("SAM-3D command is required via --command or REAL2SIM_SAM3D_COMMAND")
output_dir.mkdir(parents=True, exist_ok=True)
objects: list[dict[str, Any]] = []
jobs: list[dict[str, Any]] = []
for index, task in enumerate(tasks[:max_objects]):
if not isinstance(task, dict):
continue
job = prepare_sam3d_task(package_root=package_root, output_dir=output_dir, task=task, index=index)
jobs.append(job)
if dry_run:
continue
command = render_command(command_template, job["context"])
completed = subprocess.run(command, cwd=REPO_ROOT, check=False)
if completed.returncode != 0:
raise SystemExit(f"SAM-3D command failed for {job['id']} with exit code {completed.returncode}")
objects.append(production_object_from_job(job, package_root=package_root))
payload = {
"format": "gs-playground-real2sim-production-objects-v1",
"objects": objects,
"quality": {
"object_count": len(objects),
"source_task_count": len(tasks),
"planned_job_count": len(jobs),
"production": bool(objects) and len(objects) == len(jobs),
"dry_run": bool(dry_run),
},
"sam3d_jobs": [
{
"id": job["id"],
"label": job["label"],
"instance_json": job["instance_json"],
"image": job["image"],
"mask": job["mask"],
"output_dir": job["output_dir"],
}
for job in jobs
],
}
write_json(output_objects_json, payload)
return {"objects_json": output_objects_json.as_posix(), **payload["quality"]}
def prepare_sam3d_task(*, package_root: Path, output_dir: Path, task: dict[str, Any], index: int) -> dict[str, Any]:
task_id = str(task.get("id") or f"task_{index:03d}")
label = best_prompt_label(task)
name = f"{index:02d}_{safe_token(label)}"
object_dir = output_dir / name
object_dir.mkdir(parents=True, exist_ok=True)
crop_path = package_root / str(task["crop"])
mask_path = package_root / str(task["mask"])
rgb = read_rgb(crop_path)
mask = read_gray(mask_path)
local_mask = local_crop_mask(mask, task.get("bbox_xyxy"), rgb.shape[:2])
image_out = object_dir / "sam3d_image.ppm"
mask_out = object_dir / "sam3d_mask.pgm"
write_rgb(image_out, rgb)
write_gray(mask_out, local_mask.astype(np.float32))
instance_json = object_dir / "instance.json"
write_json(
instance_json,
{
"id": task_id,
"label": label,
"bbox_xyxy": task.get("bbox_xyxy"),
"prompt_candidates": task.get("prompt_candidates"),
"source_task": task,
},
)
gaussian = object_dir / "gaussian.ply"
mesh = object_dir / "mesh.obj"
pose = object_dir / "pose.json"
cameras = package_root / "artifacts" / "cameras_json.json"
context = {
"image": image_out,
"mask": mask_out,
"instance_json": instance_json,
"object_dir": object_dir,
"object_ply": gaussian,
"object_mesh": mesh,
"object_pose": pose,
"cameras_json": cameras if cameras.is_file() else "",
}
return {
"id": task_id,
"name": name,
"label": label,
"image": image_out.as_posix(),
"mask": mask_out.as_posix(),
"instance_json": instance_json.as_posix(),
"output_dir": object_dir.as_posix(),
"gaussian": gaussian.as_posix(),
"mesh": mesh.as_posix(),
"pose": pose.as_posix(),
"context": context,
}
def render_command(template: str, context: dict[str, Any]) -> list[str]:
rendered = template.format_map({key: str(value) for key, value in context.items()})
return shlex.split(rendered)
def production_object_from_job(job: dict[str, Any], *, package_root: Path) -> dict[str, Any]:
gaussian = require_file(Path(job["gaussian"]), "SAM-3D object gaussian")
mesh = require_file(Path(job["mesh"]), "SAM-3D object mesh")
pose_path = require_file(Path(job["pose"]), "SAM-3D object pose")
pose = json.loads(pose_path.read_text(encoding="utf-8"))
metadata = Path(job["output_dir"]) / "sam3d_metadata.json"
return {
"name": job["name"],
"label": job["label"],
"gaussian_ply": gaussian.as_posix(),
"mesh_obj": mesh.as_posix(),
"pose": pose,
"dynamic": True,
"source_instance_id": f"sam3d_{job['id']}",
"provisional": False,
"reconstruction_quality": {
"reconstruction_source": "sam3d_objects",
"commercial_asset_grade": True,
"has_gaussian_asset": True,
"has_mesh_asset": True,
"has_pose": True,
"sam3d_metadata": metadata.as_posix() if metadata.is_file() else None,
},
}
def require_file(path: Path, label: str) -> Path:
if not path.is_file() or path.stat().st_size <= 0:
raise SystemExit(f"missing {label}: {path}")
return path
def local_crop_mask(global_mask: np.ndarray, bbox: Any, crop_shape: tuple[int, int]) -> np.ndarray:
if not isinstance(bbox, list) or len(bbox) != 4:
return np.ones(crop_shape, dtype=bool)
x0, y0, x1, y1 = [int(value) for value in bbox]
cropped = global_mask[max(0, y0) : max(0, y1), max(0, x0) : max(0, x1)] > 0.5
if cropped.shape != crop_shape:
return np.ones(crop_shape, dtype=bool)
return cropped
def best_prompt_label(task: dict[str, Any]) -> str:
prompts = task.get("prompt_candidates")
if isinstance(prompts, list) and prompts:
return str(prompts[0])
return "movable_object"
def safe_token(value: str) -> str:
return "".join(ch if ch.isalnum() else "_" for ch in value.lower()).strip("_") or "object"
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,372 @@
#!/usr/bin/env python3
"""Desktop viewer for a generated Real2Sim MotrixSim scene package.
The viewer can inspect only the MJCF scene, or load a Real2Sim package config
and render the corresponding 3D Gaussian assets without attaching a robot.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
import time
from pathlib import Path
os.environ.setdefault("SDL_AUDIODRIVER", "dummy")
os.environ.setdefault("ALSOFT_DRIVERS", "null")
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "8.9")
if "XDG_RUNTIME_DIR" not in os.environ:
runtime_dir = Path(f"/run/user/{os.getuid()}")
if runtime_dir.is_dir():
os.environ["XDG_RUNTIME_DIR"] = runtime_dir.as_posix()
sys.path[:] = [
entry
for entry in sys.path
if not entry
or (
Path(entry).name != "speedy-splat"
and "real2sim_paper_runtime/speedy-splat" not in Path(entry).as_posix()
)
]
import numpy as np
from motrixsim import SceneData, msd, run
from motrixsim.render import Layout, RenderApp, RenderSettings
REPO_ROOT = Path(__file__).resolve().parents[1]
if REPO_ROOT.as_posix() not in sys.path:
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.camera import load_anysplat_source_camera
def main() -> int:
parser = argparse.ArgumentParser(description="Open a Real2Sim scene package without attaching a robot")
parser.add_argument("--config", type=Path, default=None, help="Path to a generated configs/*_real2sim.json")
parser.add_argument("--scene", type=Path, default=None, help="Path to the generated mjcf/scene.xml")
parser.add_argument(
"--camera",
default="overview",
help="Camera name from scene.xml to use as the main view; use 'system' for the interactive free camera",
)
parser.add_argument("--no-gs", action="store_true", help="Disable 3DGS preview widgets and load only the Motrix/MJCF scene")
parser.add_argument("--gs-width", type=int, default=640, help="3DGS camera widget width")
parser.add_argument("--gs-height", type=int, default=480, help="3DGS camera widget height")
parser.add_argument("--gs-widget-left", type=int, default=10, help="Left offset for 3DGS preview widgets")
parser.add_argument("--gs-widget-top", type=int, default=10, help="Top offset for 3DGS preview widgets")
parser.add_argument("--gs-render-fps", type=float, default=12.0, help="Maximum 3DGS widget render refresh rate; use 0 for every frame")
parser.add_argument("--static-gs-preview", action="store_true", help="Render 3DGS preview widgets once, then leave interaction to MotrixRender")
parser.add_argument("--viewer-fps", type=float, default=60.0, help="MotrixRender window refresh rate")
parser.add_argument("--frame-sleep-ms", type=float, default=0.0, help="Extra sleep after each render frame to keep desktop interaction responsive")
parser.add_argument("--show-source-gs", action="store_true", help="Render the AnySplat source camera from camera_artifacts.cameras_json")
parser.add_argument("--source-gs-only", action="store_true", help="Use the source-camera 3DGS render as the only 3DGS preview widget")
parser.add_argument("--source-camera-json", type=Path, default=None, help="Override source-view cameras.json artifact")
parser.add_argument("--save-source-gs", type=Path, default=None, help="Optional PNG path for the rendered source-camera 3DGS view")
parser.add_argument("--save-main-gs", type=Path, default=None, help="Optional PNG path for the rendered main-camera 3DGS view")
parser.add_argument("--show-system-gs", action="store_true", help="Also render a 3DGS view from the interactive camera")
parser.add_argument("--exit-after-static-preview", action="store_true", help="Exit after static 3DGS preview images have been rendered and saved")
parser.add_argument("--no-sync", action="store_true", help="Disable real-time sync")
args = parser.parse_args()
config = load_config(args.config)
scene_path = resolve_scene_path(args.scene, config, args.config)
scene_gaussians = {} if args.no_gs else collect_scene_gaussians(config, args.config)
camera_artifacts = collect_camera_artifacts(config, args.config)
source_camera_json = args.source_camera_json or camera_artifacts.get("cameras_json")
gaussian_links = {
str(gs_name): str(link_name)
for gs_name, link_name in config.get("gaussian_links", {}).items()
}
world = msd.from_file(scene_path.as_posix())
model = world.build()
data = SceneData(model)
render_target_fps = max(int(round(float(args.viewer_fps))), 1)
with RenderApp(fps=render_target_fps) as render:
print(f"Launching Real2Sim scene viewer: {scene_path}", flush=True)
render.launch(model, render_settings=RenderSettings.performance())
for group_id in range(6):
try:
render.opt.set_group_vis(group_id, True)
except Exception:
pass
use_system_camera = args.camera.lower() in {"system", "free", "interactive"}
camera = None if use_system_camera else find_camera(model, args.camera)
camera_id = None if use_system_camera else find_camera_id(model, args.camera)
if use_system_camera:
render.set_main_camera(None)
seed_camera = find_camera(model, "overview") or (model.cameras[0] if len(model.cameras) else None)
if seed_camera is not None:
try:
render.system_camera.pose = seed_camera.get_pose(data).astype(np.float32)
except Exception:
pass
print("Using system camera", flush=True)
elif camera is not None:
render.set_main_camera(camera)
print(f"Using camera: {args.camera}", flush=True)
gs_renderer = None
gs_image = None
gs_source_image = None
gs_system_image = None
gs_debug_printed = False
source_rendered = False
source_render_failed = False
source_camera = None
if scene_gaussians:
print(f"Loading 3DGS assets: {', '.join(sorted(scene_gaussians))}", flush=True)
from gaussian_renderer import GSRendererMotrixSim
gs_renderer = GSRendererMotrixSim(scene_gaussians, model)
apply_gaussian_links(gs_renderer, model, gaussian_links)
widget_top = int(args.gs_widget_top)
widget_left = int(args.gs_widget_left)
if args.source_gs_only:
args.show_source_gs = True
if args.show_source_gs or args.save_source_gs is not None:
if source_camera_json is None:
print("Source 3DGS view requested but no camera_artifacts.cameras_json was found.", flush=True)
elif source_camera_json.is_file():
source_camera = load_anysplat_source_camera(source_camera_json)
gs_source_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8))
if args.show_source_gs:
render.widgets.create_image_widget(
gs_source_image,
layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height),
)
widget_top += args.gs_height + 10
print(f"Source 3DGS camera ready: {source_camera_json}", flush=True)
else:
print(f"Source camera JSON is missing: {source_camera_json}", flush=True)
if not args.source_gs_only:
gs_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8))
render.widgets.create_image_widget(
gs_image,
layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height),
)
widget_top += args.gs_height + 10
if args.show_system_gs:
gs_system_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8))
render.widgets.create_image_widget(
gs_system_image,
layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height),
)
print("3DGS renderer ready.", flush=True)
print(f"Scene viewer ready. Bodies: {len(model.body_names)} Links: {len(model.link_names)}", flush=True)
def phys_step() -> None:
return None
last_gs_render_at = 0.0
gs_preview_rendered = False
gs_render_interval = 0.0 if args.gs_render_fps <= 0 else 1.0 / max(float(args.gs_render_fps), 1e-6)
def render_step() -> bool:
nonlocal gs_debug_printed, source_rendered, source_render_failed, last_gs_render_at, gs_preview_rendered
if render.input.is_key_just_pressed("esc") or render.input.is_key_just_pressed("escape"):
return False
if gs_renderer is not None:
now = time.monotonic()
should_render_gs = (
not (args.static_gs_preview and gs_preview_rendered)
and (gs_render_interval <= 0.0 or (now - last_gs_render_at) >= gs_render_interval)
)
if source_camera is not None and gs_source_image is not None and not source_rendered and not source_render_failed:
try:
gs_renderer.update_gaussians(data)
source_rgb, _source_depth = gs_renderer.render_batch(
source_camera.position.reshape(1, 3),
source_camera.xmat.reshape(1, 9),
args.gs_height,
args.gs_width,
np.array([source_camera.fovy_degrees], dtype=np.float32),
y_up=source_camera.y_up,
)
gs_source_image.pixels = rgb_to_uint8(source_rgb[0])
if args.save_source_gs is not None:
args.save_source_gs.parent.mkdir(parents=True, exist_ok=True)
gs_source_image.save_to_disk(args.save_source_gs.as_posix())
print(f"Saved source 3DGS render: {args.save_source_gs}", flush=True)
source_rendered = True
print("3DGS rendered source camera.", flush=True)
if args.source_gs_only and args.static_gs_preview and not gs_preview_rendered:
gs_preview_rendered = True
print("3DGS static preview rendered.", flush=True)
except Exception as exc:
source_render_failed = True
print(f"Source 3DGS render failed: {exc}", flush=True)
if should_render_gs and not args.source_gs_only:
last_gs_render_at = now
gs_renderer.update_gaussians(data)
render_ids = []
if camera_id is not None:
render_ids.append(camera_id)
if args.show_system_gs:
render_ids.append(-1)
if render_ids:
results = gs_renderer.render(
model,
data,
render_ids,
args.gs_width,
args.gs_height,
system_camera=render.system_camera,
)
if not gs_debug_printed:
print(f"3DGS rendered camera ids: {sorted(results.keys())}", flush=True)
gs_debug_printed = True
if camera_id in results and gs_image is not None:
gs_image.pixels = rgb_to_uint8(results[camera_id][0])
if args.save_main_gs is not None:
args.save_main_gs.parent.mkdir(parents=True, exist_ok=True)
gs_image.save_to_disk(args.save_main_gs.as_posix())
print(f"Saved main 3DGS render: {args.save_main_gs}", flush=True)
if -1 in results and gs_system_image is not None:
gs_system_image.pixels = rgb_to_uint8(results[-1][0])
if args.static_gs_preview:
gs_preview_rendered = True
print("3DGS static preview rendered.", flush=True)
should_exit_after_preview = args.exit_after_static_preview and args.static_gs_preview and gs_preview_rendered
render.sync(data)
if args.frame_sleep_ms > 0:
time.sleep(float(args.frame_sleep_ms) / 1000.0)
if should_exit_after_preview:
# Motrix's render loop can ignore a false-y callback return on
# the Wuying headless display. This flag is only for automated
# static screenshot capture, so exit explicitly after flushing.
sys.stdout.flush()
sys.stderr.flush()
os._exit(0)
return True
render_fps = 1000000.0 if args.no_sync else max(float(args.viewer_fps), 1.0)
run.render_loop(model.options.timestep, render_fps, phys_step, render_step)
return 0
def find_camera(model, name: str):
try:
return model.cameras[name]
except Exception:
pass
for camera in model.cameras:
if getattr(camera, "name", None) == name:
return camera
return model.cameras[0] if len(model.cameras) else None
def find_camera_id(model, name: str) -> int | None:
for idx, camera in enumerate(model.cameras):
if getattr(camera, "name", None) == name:
return idx
return 0 if len(model.cameras) else None
def load_config(config_path: Path | None) -> dict:
if config_path is None:
return {}
with config_path.open("r", encoding="utf-8") as f:
config = json.load(f)
if not isinstance(config, dict):
raise ValueError(f"Real2Sim viewer config must be a JSON object: {config_path}")
return config
def resolve_config_path(path: str | Path, config_path: Path | None) -> Path:
resolved = Path(path)
if resolved.is_absolute():
return resolved
if config_path is not None:
return (config_path.resolve().parent / resolved).resolve()
return resolved.resolve()
def resolve_scene_path(scene_arg: Path | None, config: dict, config_path: Path | None) -> Path:
scene = scene_arg if scene_arg is not None else config.get("scene")
if not scene:
raise ValueError("Provide --scene or a --config containing a scene path")
return resolve_config_path(scene, config_path)
def collect_scene_gaussians(config: dict, config_path: Path | None) -> dict[str, str]:
scene_gaussians = config.get("scene_gaussians", {})
if not isinstance(scene_gaussians, dict):
raise ValueError("scene_gaussians must be an object mapping Gaussian names to PLY paths")
gaussians: dict[str, str] = {}
missing: list[Path] = []
for name, path in scene_gaussians.items():
ply = resolve_config_path(path, config_path)
if ply.exists():
gaussians[str(name)] = ply.as_posix()
else:
missing.append(ply)
if missing:
print("Skipping missing 3DGS assets: " + ", ".join(path.as_posix() for path in missing), flush=True)
return gaussians
def collect_camera_artifacts(config: dict, config_path: Path | None) -> dict[str, Path]:
raw_artifacts = config.get("camera_artifacts", {})
if not isinstance(raw_artifacts, dict):
raise ValueError("camera_artifacts must be an object mapping artifact names to paths")
artifacts: dict[str, Path] = {}
missing: list[Path] = []
for name, path in raw_artifacts.items():
artifact = resolve_config_path(path, config_path)
if artifact.exists():
artifacts[str(name)] = artifact
else:
missing.append(artifact)
if missing:
print("Skipping missing camera artifacts: " + ", ".join(path.as_posix() for path in missing), flush=True)
return artifacts
def apply_gaussian_links(gs_renderer, model, gaussian_links: dict[str, str]) -> None:
if not gaussian_links:
return
link_ids = {name: idx for idx, name in enumerate(model.link_names)}
objects_info = []
body_ids = []
skipped = []
for gs_name, link_name in gaussian_links.items():
if gs_name not in gs_renderer.gaussian_start_indices:
skipped.append(f"{gs_name}:missing-gaussian")
continue
if link_name not in link_ids:
skipped.append(f"{gs_name}:{link_name}:missing-link")
continue
objects_info.append((gs_name, gs_renderer.gaussian_start_indices[gs_name], gs_renderer.gaussian_end_indices[gs_name]))
body_ids.append(link_ids[link_name])
if skipped:
print("Skipping Gaussian links: " + ", ".join(skipped), flush=True)
if not objects_info:
return
gs_renderer.gs_idx_start = np.array([start for _, start, _ in objects_info])
gs_renderer.gs_idx_end = np.array([end for _, _, end in objects_info])
gs_renderer.gs_body_ids = np.array(body_ids)
gs_renderer.set_objects_mapping(objects_info)
print(f"Bound 3DGS assets to MJCF links: {', '.join(name for name, _, _ in objects_info)}", flush=True)
def rgb_to_uint8(rgb_tensor) -> np.ndarray:
rgb = rgb_tensor.detach().cpu().numpy()
if rgb.dtype == np.uint8:
return rgb
return np.clip(rgb * 255.0, 0, 255).astype(np.uint8)
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Sweep MotrixSim scene-completion parameters for a Real2Sim package.
The sweep keeps the reconstructed 3DGS and textured scene mesh fixed, varies
the photo-completion backdrop margin, packages each candidate, runs the MuJoCo
offscreen verifier, and ranks candidates by the verifier's render-quality
score. It is intended for same-capture candidate comparison, not as an absolute
commercial-quality metric.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
def main() -> int:
parser = argparse.ArgumentParser(description="Sweep Real2Sim Motrix scene-completion backdrop margins")
parser.add_argument("--source", type=Path, required=True, help="Original capture directory or source path")
parser.add_argument("--scene-ply", type=Path, required=True)
parser.add_argument("--scene-mesh", type=Path, required=True)
parser.add_argument("--cameras-json", type=Path, required=True)
parser.add_argument("--preview-image", type=Path, required=True)
parser.add_argument("--mesh-cleanup-report", type=Path, required=True)
parser.add_argument("--texture-bake-report", type=Path, required=True)
parser.add_argument("--completion-report", type=Path, required=True)
parser.add_argument("--completion-texture", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--name-prefix", default="real2sim_scene_completion_sweep")
parser.add_argument("--margins", default="0,0.15,0.25,0.35,0.55", help="Comma-separated backdrop margins to test")
parser.add_argument("--scene-faces", type=int, default=30_000, help="Default scene mesh LOD face budget")
parser.add_argument("--scene-face-options", default="", help="Comma-separated scene mesh LOD face budgets to sweep")
parser.add_argument("--min-scene-vertices", type=int, default=100_000)
parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"])
parser.add_argument("--prompt", default="photogrammetry textured scene with swept photo completion backdrop")
parser.add_argument("--python", default=sys.executable)
parser.add_argument("--archive", action="store_true", help="Write a .tar.gz for each candidate package")
parser.add_argument("--skip-existing", action="store_true", help="Reuse package and verifier output directories when present")
args = parser.parse_args()
margins = parse_margins(args.margins)
face_options = parse_int_options(args.scene_face_options) if args.scene_face_options else [int(args.scene_faces)]
args.output_root.mkdir(parents=True, exist_ok=True)
candidates: list[dict[str, Any]] = []
for margin in margins:
for scene_faces in face_options:
candidates.append(run_candidate(args, margin, scene_faces))
ranked = rank_candidates(candidates)
report = {
"format": "gs-playground-real2sim-scene-completion-sweep-v1",
"target_quality": "polycam_realitycapture_like_space",
"margins": margins,
"scene_face_options": face_options,
"best": ranked[0] if ranked else None,
"candidates": ranked,
"notes": [
"Scores are comparable only within the same capture, cameras, render size, and verifier version.",
"A high score does not prove Polycam/RealityCapture-level fidelity; inspect contact sheets before accepting.",
],
}
report_path = args.output_root / "sweep_report.json"
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(json.dumps(report, indent=2, ensure_ascii=False))
return 0
def parse_margins(raw: str) -> list[float]:
values: list[float] = []
for item in raw.split(","):
item = item.strip()
if not item:
continue
value = float(item)
if value < 0:
raise SystemExit(f"margin must be non-negative: {value}")
values.append(value)
if not values:
raise SystemExit("at least one margin is required")
return values
def parse_int_options(raw: str) -> list[int]:
values: list[int] = []
for item in raw.split(","):
item = item.strip()
if not item:
continue
value = int(item)
if value <= 0:
raise SystemExit(f"scene face option must be positive: {value}")
values.append(value)
if not values:
raise SystemExit("at least one scene face option is required")
return values
def run_candidate(args: argparse.Namespace, margin: float, scene_faces: int) -> dict[str, Any]:
token = candidate_token(margin, scene_faces)
package_dir = args.output_root / f"{args.name_prefix}_{token}_package"
verify_dir = args.output_root / f"{args.name_prefix}_{token}_verify"
report_path = verify_dir / "report.json"
if not (args.skip_existing and report_path.is_file()):
if package_dir.exists():
shutil.rmtree(package_dir)
if verify_dir.exists():
shutil.rmtree(verify_dir)
verify_dir.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env.update(
{
"REAL2SIM_SCENE_COMPLETION_REPORT": args.completion_report.as_posix(),
"REAL2SIM_SCENE_COMPLETION_TEXTURE": args.completion_texture.as_posix(),
"REAL2SIM_SCENE_COMPLETION_BACKDROP_MARGIN": str(margin),
"REAL2SIM_MOTRIX_SCENE_VISUAL": "photogrammetry",
"REAL2SIM_MOTRIX_PREVIEW_MESH_FACES": str(max(int(scene_faces), 1)),
}
)
package_cmd = [
args.python,
"-m",
"real2sim",
"package-existing",
"--source",
args.source.as_posix(),
"--scene-ply",
args.scene_ply.as_posix(),
"--scene-mesh",
args.scene_mesh.as_posix(),
"--cameras-json",
args.cameras_json.as_posix(),
"--preview-image",
args.preview_image.as_posix(),
"--mesh-cleanup-report",
args.mesh_cleanup_report.as_posix(),
"--texture-bake-report",
args.texture_bake_report.as_posix(),
"--output",
package_dir.as_posix(),
"--prompt",
args.prompt,
"--robot",
args.robot,
]
run(package_cmd, env=env, cwd=REPO_ROOT)
verify_cmd = [
args.python,
(REPO_ROOT / "scripts" / "real2sim_verify_motrix_package.py").as_posix(),
"--package",
package_dir.as_posix(),
"--min-scene-vertices",
str(int(args.min_scene_vertices)),
"--mujoco-offscreen-render-dir",
verify_dir.as_posix(),
"--visual-report-dir",
verify_dir.as_posix(),
]
with report_path.open("w", encoding="utf-8") as handle:
run(verify_cmd, env={"MUJOCO_GL": "egl", **env}, cwd=REPO_ROOT, stdout=handle)
if args.archive:
archive_path = package_dir.with_suffix(".tar.gz")
run(["tar", "-C", package_dir.parent.as_posix(), "-czf", archive_path.as_posix(), package_dir.name], cwd=REPO_ROOT)
verification = json.loads(report_path.read_text(encoding="utf-8"))
return candidate_summary(margin, scene_faces, package_dir, verify_dir, verification)
def run(
cmd: list[str],
*,
cwd: Path,
env: dict[str, str] | None = None,
stdout: Any | None = None,
) -> None:
process_env = os.environ.copy()
if env:
process_env.update(env)
subprocess.run(cmd, cwd=cwd, env=process_env, stdout=stdout, check=True)
def candidate_summary(
margin: float,
scene_faces: int,
package_dir: Path,
verify_dir: Path,
verification: dict[str, Any],
) -> dict[str, Any]:
renders = verification.get("mujoco_offscreen_render", {}).get("renders", [])
scores = [
float(render.get("quality", {}).get("score", 0.0))
for render in renders
if isinstance(render, dict)
]
overview = next((render for render in renders if render.get("camera") == "overview"), {}) if isinstance(renders, list) else {}
closeup = next((render for render in renders if render.get("camera") == "closeup"), {}) if isinstance(renders, list) else {}
avg_score = sum(scores) / len(scores) if scores else 0.0
return {
"margin": float(margin),
"scene_faces": int(scene_faces),
"score": float(avg_score),
"package": package_dir.as_posix(),
"verify_dir": verify_dir.as_posix(),
"contact_sheet": (verify_dir / "visual_acceptance_contact_sheet.png").as_posix(),
"overview": render_quality_summary(overview),
"closeup": render_quality_summary(closeup),
}
def render_quality_summary(render: dict[str, Any]) -> dict[str, Any]:
quality = render.get("quality") if isinstance(render, dict) else {}
return {
"path": render.get("path") if isinstance(render, dict) else None,
"score": float(quality.get("score", 0.0)) if isinstance(quality, dict) else 0.0,
"black_ratio": float(quality.get("black_ratio", 0.0)) if isinstance(quality, dict) else 0.0,
"high_saturation_ratio": float(quality.get("high_saturation_ratio", 0.0)) if isinstance(quality, dict) else 0.0,
}
def rank_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
return sorted(
candidates,
key=lambda item: (
float(item.get("score", 0.0)),
int(item.get("scene_faces", 0)),
-float(item.get("margin", 0.0)),
),
reverse=True,
)
def margin_token(margin: float) -> str:
return f"margin_{margin:.3f}".replace(".", "p").replace("-", "m")
def candidate_token(margin: float, scene_faces: int) -> str:
return f"{margin_token(margin)}_faces_{int(scene_faces)}"
if __name__ == "__main__":
raise SystemExit(main())

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,108 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat >&2 <<'USAGE'
Usage:
scripts/real2sim_wuying_launch_package.sh --archive /path/to/real2sim_scene.tar.gz [--dest /tmp/real2sim_import] [--with-gs]
Extract a generated one-image Real2Sim Motrix package on the Wuying desktop and
launch the visible MotrixSim viewer.
USAGE
}
archive=""
dest="/tmp/real2sim_import"
with_gs=0
while [[ $# -gt 0 ]]; do
case "$1" in
--archive)
archive="${2:-}"
shift 2
;;
--dest)
dest="${2:-}"
shift 2
;;
--with-gs)
with_gs=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage
exit 2
;;
esac
done
if [[ -z "${archive}" || ! -f "${archive}" ]]; then
echo "--archive must point to an existing .tar.gz package" >&2
exit 2
fi
mkdir -p "${dest}"
tar -xzf "${archive}" -C "${dest}"
package_dir="$(find "${dest}" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1)"
if [[ -z "${package_dir}" || ! -f "${package_dir}/configs/go2_real2sim.json" ]]; then
echo "Extracted package missing configs/go2_real2sim.json under ${dest}" >&2
exit 1
fi
recommended_camera=""
if [[ -f "${package_dir}/motrix_import.json" ]]; then
recommended_camera="$(python3 - "${package_dir}/motrix_import.json" <<'PY' || true
import json
import sys
try:
data = json.load(open(sys.argv[1], encoding="utf-8"))
except Exception:
data = {}
print(data.get("recommended_camera", ""))
PY
)"
fi
viewer_camera="overview"
if [[ "${recommended_camera}" == "source_3dgs_preview" ]]; then
viewer_camera="closeup"
fi
args=(--config "${package_dir}/configs/go2_real2sim.json" --camera "${viewer_camera}" --gs-width 900 --gs-height 700 --static-gs-preview)
if [[ "${with_gs}" -eq 1 ]]; then
args+=(--show-source-gs)
if [[ "${recommended_camera}" == "source_3dgs_preview" ]]; then
args+=(--source-gs-only)
fi
args=(--with-gs "${args[@]}")
fi
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
runtime_repo="${REAL2SIM_WUYING_RUNTIME_REPO:-$(cd "${script_dir}/.." && pwd)}"
if [[ ! -x "${runtime_repo}/scripts/real2sim_wuying_viewer.sh" || ! -x "${runtime_repo}/.venv/bin/python" ]]; then
for candidate in \
/opt/gs_playground_real2sim_48q \
/opt/gs_playground-real2sim_48q \
/opt/gs_playground \
/workspace/gs_playground \
/root/gs_playground; do
if [[ -x "${candidate}/scripts/real2sim_wuying_viewer.sh" && -x "${candidate}/.venv/bin/python" ]]; then
runtime_repo="${candidate}"
break
fi
done
fi
if [[ ! -x "${runtime_repo}/scripts/real2sim_wuying_viewer.sh" || ! -x "${runtime_repo}/.venv/bin/python" ]]; then
echo "Could not find a Real2Sim runtime repo with scripts/real2sim_wuying_viewer.sh and .venv/bin/python." >&2
echo "Set REAL2SIM_WUYING_RUNTIME_REPO=/path/to/gs_playground on the Wuying desktop." >&2
exit 1
fi
echo "Launching MotrixSim Real2Sim package: ${package_dir}"
echo "Runtime repo: ${runtime_repo}"
cd "${runtime_repo}"
exec scripts/real2sim_wuying_viewer.sh "${args[@]}"

View File

@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Runtime wiring for the 48 GiB Wuying Real2Sim paper-alignment machine.
#
# Usage:
# source scripts/real2sim_wuying_paper_env.sh
# python -m real2sim check --paper-strict --config configs/real2sim.external.example.json --backend external
set -euo pipefail
export REAL2SIM_PROJECT_ROOT="${REAL2SIM_PROJECT_ROOT:-/opt/gs_playground_real2sim_48q}"
export REAL2SIM_RUNTIME_ROOT="${REAL2SIM_RUNTIME_ROOT:-/opt/real2sim_paper_runtime}"
export REAL2SIM_ANYSPLAT_ROOT="${REAL2SIM_ANYSPLAT_ROOT:-${REAL2SIM_RUNTIME_ROOT}/AnySplat}"
export REAL2SIM_SAM2_ROOT="${REAL2SIM_SAM2_ROOT:-${REAL2SIM_RUNTIME_ROOT}/sam2}"
export REAL2SIM_SAM3D_ROOT="${REAL2SIM_SAM3D_ROOT:-${REAL2SIM_RUNTIME_ROOT}/sam-3d-objects}"
export REAL2SIM_SPEEDY_SPLAT_ROOT="${REAL2SIM_SPEEDY_SPLAT_ROOT:-${REAL2SIM_RUNTIME_ROOT}/speedy-splat}"
export REAL2SIM_ANY_SPLAT_PYTHON="${REAL2SIM_ANY_SPLAT_PYTHON:-${REAL2SIM_RUNTIME_ROOT}/envs/anysplat/bin/python}"
export REAL2SIM_SAM3D_PYTHON="${REAL2SIM_SAM3D_PYTHON:-${REAL2SIM_RUNTIME_ROOT}/envs/sam3d-objects/bin/python}"
export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda-12.1}"
export CONDA_PREFIX="${CONDA_PREFIX:-${CUDA_HOME}}"
export LIDRA_SKIP_INIT="${LIDRA_SKIP_INIT:-1}"
export PATH="${CUDA_HOME}/bin:${PATH}"
export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}"
export REAL2SIM_SAM2_CHECKPOINT="${REAL2SIM_SAM2_CHECKPOINT:-${REAL2SIM_SAM2_ROOT}/sam2.1_hiera_large.pt}"
export REAL2SIM_SAM2_MODEL_CFG="${REAL2SIM_SAM2_MODEL_CFG:-configs/sam2.1/sam2.1_hiera_l.yaml}"
export REAL2SIM_SAM3D_CONFIG="${REAL2SIM_SAM3D_CONFIG:-${REAL2SIM_SAM3D_ROOT}/checkpoints/hf/pipeline.yaml}"
_real2sim_pythonpath_parts=(
"${REAL2SIM_SAM2_ROOT}"
"${REAL2SIM_SAM3D_ROOT}/notebook"
"${REAL2SIM_SAM3D_ROOT}"
"${REAL2SIM_ANYSPLAT_ROOT}"
"${REAL2SIM_SPEEDY_SPLAT_ROOT}"
)
if [[ -n "${PYTHONPATH:-}" ]]; then
_real2sim_pythonpath_parts+=("${PYTHONPATH}")
fi
export PYTHONPATH="$(IFS=:; echo "${_real2sim_pythonpath_parts[*]}")"
unset _real2sim_pythonpath_parts
export REAL2SIM_SYNTHETIC_MULTIVIEW_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/synthetic_multiview_expand.py --image {image} --output-dir {synthetic_multiview_dir} --frames-dir {synthetic_frames_dir} --prompt {synthetic_multiview_prompt} --mode {synthetic_multiview_mode} --size 1536x1024"
export REAL2SIM_GROUNDING_DINO_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/groundingdino_hf_detect.py --image {image} --prompt {prompt} --output {detections_json} --max-detections 12"
export REAL2SIM_SAM2_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/sam2_segment.py --image {image} --detections {detections_json} --masks-dir {masks_dir} --output {instances_json}"
export REAL2SIM_INSTANCE_REFINEMENT_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/instance_refine.py --instances {instances_json} --output {instances_json} --image {image} --merge-saturated-accessories --accessory-search-margin-ratio 0.25 --accessory-contact-passes 4 --accessory-saturation-threshold 0.45 --accessory-max-area-growth 0.15 --accessory-min-component-area-ratio 0.001 --accessory-max-components 8"
export REAL2SIM_LAMA_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/lama_inpaint.py --image {image} --instances {instances_json} --output {background_image} --prompt {prompt} --work-dir {work_dir} --redetect-command-env REAL2SIM_GROUNDING_DINO_COMMAND --segment-command-env REAL2SIM_SAM2_COMMAND --require-redetect-loop"
export REAL2SIM_DEPTH_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/depth_anything_depth.py --image {background_image} --output {depth_map} --model depth-anything/Depth-Anything-V2-Small-hf"
export REAL2SIM_ANYSPLAT_COMMAND="${REAL2SIM_ANY_SPLAT_PYTHON} scripts/real2sim_external/anysplat_scene.py --image {background_image} --frames-dir {frames_dir} --output-dir {scene_reconstruction_dir} --ply {scene_ply} --mesh {scene_mesh} --cameras {cameras_json} --depth {depth_map} --max-views 8"
export REAL2SIM_SAM3D_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/sam3d_objects_reconstruct.py --image {image} --mask {mask} --instance {instance_json} --output-dir {object_dir} --ply {object_ply} --mesh {object_mesh} --pose {object_pose} --cameras {cameras_json} --match-camera-preprocess --object-centric-preprocess --object-preprocess-size 896 --object-crop-margin 0.12 --canonicalize-gaussian-to-mesh --remove-video-overlay --texture-baking --texture-render-backend gsplat --texture-baking-engine pytorch3d"
export REAL2SIM_POSE_ALIGNMENT_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/depth_mask_pose_align.py --assets {aligned_assets_json} --instances {instances_json} --depth {depth_map} --cameras {cameras_json} --work-dir {work_dir} --output {aligned_assets_json}"
export REAL2SIM_SPEEDY_SPLAT_COMMAND="${REAL2SIM_ANY_SPLAT_PYTHON} scripts/real2sim_external/speedy_splat_prune.py --input {input} --output {output} --image {source_image} --cameras {cameras_json} --keep-ratio {keep_ratio}"

View File

@ -0,0 +1,200 @@
#!/usr/bin/env bash
# Launch the generated Real2Sim viewer on the visible Wuying desktop.
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${repo_root}"
foreground=0
with_gs=0
force_render_app=0
package_dir="${REAL2SIM_WUYING_PACKAGE:-}"
config_path="${REAL2SIM_WUYING_CONFIG:-}"
scene_path="${REAL2SIM_WUYING_SCENE:-}"
while [[ "$#" -gt 0 ]]; do
case "${1}" in
--foreground)
foreground=1
shift
;;
--with-gs)
with_gs=1
force_render_app=1
shift
;;
--render-app)
force_render_app=1
shift
;;
--native)
force_render_app=0
shift
;;
--package)
package_dir="${2:?--package requires a package directory}"
shift 2
;;
--config)
config_path="${2:?--config requires a config JSON path}"
force_render_app=1
shift 2
;;
--scene)
scene_path="${2:?--scene requires an MJCF scene path}"
shift 2
;;
--stop)
pkill -f "real2sim_scene_viewer[.]py" || true
pkill -f "motrixsim[.]interactive_viewer" || true
echo "Stopped Real2Sim viewer processes."
exit 0
;;
*)
break
;;
esac
done
unset PYTHONPATH
export SDL_AUDIODRIVER="${SDL_AUDIODRIVER:-dummy}"
export ALSOFT_DRIVERS="${ALSOFT_DRIVERS:-null}"
export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-8.9}"
if [[ -z "${XDG_RUNTIME_DIR:-}" && -d "/run/user/$(id -u)" ]]; then
export XDG_RUNTIME_DIR="/run/user/$(id -u)"
fi
choose_display() {
local first_ok=""
local candidate
local seen=" "
for candidate in "${DISPLAY:-}" :1 :92; do
[[ -n "${candidate}" ]] || continue
[[ "${seen}" == *" ${candidate} "* ]] && continue
seen+="${candidate} "
DISPLAY="${candidate}" timeout 2s xdpyinfo >/dev/null 2>&1 || continue
if [[ -z "${first_ok}" ]]; then
first_ok="${candidate}"
fi
if DISPLAY="${candidate}" timeout 2s xwininfo -root -tree 2>/dev/null | grep -Eq 'gnome-shell|gnome-terminal|MotrixRender'; then
printf '%s\n' "${candidate}"
return 0
fi
done
if [[ -n "${first_ok}" ]]; then
printf '%s\n' "${first_ok}"
return 0
fi
printf '%s\n' "${DISPLAY:-:1}"
}
export DISPLAY="$(choose_display)"
echo "Launching Real2Sim viewer on DISPLAY=${DISPLAY}"
resolve_latest_package() {
find outputs -path "*/mjcf/scene.xml" -type f -printf '%T@ %h\n' 2>/dev/null \
| sort -nr \
| head -1 \
| awk '{print $2}' \
| sed 's#/mjcf$##'
}
if [[ -z "${package_dir}" && ( -z "${scene_path}" || -z "${config_path}" ) ]]; then
package_dir="$(resolve_latest_package || true)"
fi
if [[ -n "${package_dir}" ]]; then
if [[ -z "${scene_path}" ]]; then
scene_path="${package_dir%/}/mjcf/scene.xml"
fi
if [[ -z "${config_path}" ]]; then
config_path="$(find "${package_dir%/}/configs" -maxdepth 1 -name '*_real2sim.json' -type f 2>/dev/null | sort | head -1 || true)"
fi
fi
if [[ "$#" -eq 0 ]]; then
if [[ "${force_render_app}" -eq 0 ]]; then
if [[ -z "${scene_path}" ]]; then
echo "No scene path found. Pass --scene or --package." >&2
exit 1
fi
set -- \
--file="${scene_path}"
elif [[ "${with_gs}" -eq 1 ]]; then
if [[ -z "${config_path}" ]]; then
echo "No Real2Sim config found. Pass --config or --package." >&2
exit 1
fi
set -- \
--config "${config_path}" \
--camera overview \
--gs-width 320 \
--gs-height 210 \
--gs-widget-left 930 \
--gs-widget-top 10 \
--gs-render-fps 8 \
--static-gs-preview \
--viewer-fps 20 \
--frame-sleep-ms 30 \
--show-source-gs \
--show-system-gs
else
if [[ -n "${config_path}" ]]; then
set -- \
--config "${config_path}" \
--camera system \
--no-gs \
--viewer-fps 12 \
--frame-sleep-ms 55
elif [[ -n "${scene_path}" ]]; then
set -- \
--scene "${scene_path}" \
--camera system \
--no-gs \
--viewer-fps 12 \
--frame-sleep-ms 55
else
echo "No Real2Sim config or scene found. Pass --config, --scene, or --package." >&2
exit 1
fi
fi
else
force_render_app=1
parsed_args=()
if [[ -n "${config_path}" ]]; then
parsed_args+=(--config "${config_path}")
elif [[ -n "${scene_path}" ]]; then
parsed_args+=(--scene "${scene_path}")
fi
set -- "${parsed_args[@]}" "$@"
fi
pkill -f "real2sim_scene_viewer[.]py" || true
pkill -f "motrixsim[.]interactive_viewer" || true
if [[ "${foreground}" -eq 1 ]]; then
if [[ "${force_render_app}" -eq 0 ]]; then
exec .venv/bin/python -m motrixsim.interactive_viewer "$@"
fi
exec .venv/bin/python scripts/real2sim_scene_viewer.py "$@"
fi
log_path="${REAL2SIM_WUYING_VIEWER_LOG:-/tmp/real2sim_wuying_viewer.log}"
rm -f "${log_path}"
if [[ "${force_render_app}" -eq 0 ]]; then
cmd=(.venv/bin/python -m motrixsim.interactive_viewer "$@")
else
cmd=(.venv/bin/python scripts/real2sim_scene_viewer.py "$@")
fi
printf 'Command:' >"${log_path}"
printf ' %q' "${cmd[@]}" >>"${log_path}"
printf '\n' >>"${log_path}"
setsid "${cmd[@]}" >>"${log_path}" 2>&1 < /dev/null &
pid=$!
echo "Started Real2Sim viewer PID=${pid}"
echo "Log: ${log_path}"
sleep 2
if ! ps -p "${pid}" >/dev/null 2>&1; then
echo "Viewer exited early; last log lines:" >&2
tail -80 "${log_path}" >&2 || true
exit 1
fi